{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "from pathlib import Path\n",
    "import re\n",
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import shutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for path_item in SLR_wav_list:\n",
    "#     wav_path = str(path_item)\n",
    "#     shutil.move(wav_path,wav_path[0:-4]) \n",
    " "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 准备词典"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !cd data && python ../tools/prep_dict.py lexicon.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !./tools/prepare_dict.sh data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !rm -rf gmm/data/dict/\n",
    "# !mkdir gmm/data/dict/\n",
    "# !cd data &&cp lexicon.txt silence_phones.txt nonsilence_phones.txt extra_questions.txt optional_silence.txt ../gmm/data/dict\n",
    "# # !rm silence_phones.txt nonsilence_phones.txt extra_questions.txt optional_silence.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SLR85生成wav.scp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "SLR85_dir = \"/home1/meichaoyang/Dataset/feats/SLR85/hifi/train\"\n",
    "SLR_wav_list=list(Path(SLR85_dir).rglob(\"*.wav\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "SLR85_dir = \"/home1/meichaoyang/Dataset/feats/SLR85/hifi/dev\"\n",
    "SLR_wav_list=list(Path(SLR85_dir).rglob(\"*.wav\")) + SLR_wav_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "SLR_wav_list.sort(key=lambda Path: os.path.basename(os.path.splitext((str(Path)))[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2377"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(SLR_wav_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('SLR_wav.scp', 'w') as f:\n",
    "    for path_item in SLR_wav_list:\n",
    "        wav_path = str(path_item)\n",
    "        f.write(os.path.basename(os.path.splitext((wav_path))[0])+'\\t'+wav_path+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 并筛选一部分放入SLR_wav.scp.shuff文件中"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !bash tools/random_line.sh SLR_wav.scp 100 SLR_wav.scp.shuff\n",
    "!cat SLR_wav.scp > SLR_wav.scp.shuff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "> 并筛选一部分放入SLR_wav.scp.shuff文件中"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Aishell2数据处理"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 生成Aishell2的原始wav.scp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "aishell_dir = \"/home1/meichaoyang/Dataset/data_aishell2/data_aishell/wav/train\"\n",
    "aishell_wav_list=list(Path(aishell_dir).rglob(\"*.wav\"))\n",
    "aishell_wav_list.sort(key=lambda Path: os.path.basename(os.path.splitext((str(Path)))[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "aishell_scp_raw = {}\n",
    "with open('Aishell_wav_raw.scp', 'w') as f:\n",
    "    for path_item in aishell_wav_list:\n",
    "        wav_path = str(path_item)\n",
    "        aishell_scp_raw[os.path.basename(os.path.splitext((wav_path))[0])] = wav_path\n",
    "        f.write(os.path.basename(os.path.splitext((wav_path))[0])+'\\t'+wav_path+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 对满足条件的标注进行筛选"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "pattern_utt = re.compile(r'/.*\\.')\n",
    "pattern_Eng = re.compile(u'[a-zA-Z\\n]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "aishell_2_corp_raw = \"/home1/meichaoyang/Dataset/data_aishell2/data_aishell/wav/trans.txt\"\n",
    "corp_map = {}\n",
    "with open(aishell_2_corp_raw, \"r\") as f:\n",
    "    for line in f:\n",
    "        data = line.split()\n",
    "        if pattern_Eng.search(data[1]) != None: ##删除小于10和非英文标注\n",
    "            continue\n",
    "        corp_map[pattern_utt.search(data[0]).group()[1:-1]] = data[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "aishell_scp = {}\n",
    "for i in sorted(aishell_scp_raw):\n",
    "    if i not in corp_map.keys():\n",
    "        continue\n",
    "    aishell_scp[i] = aishell_scp_raw[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('aishell_2.scp', 'w') as f:\n",
    "    for i in aishell_scp:\n",
    "        f.write(i+'\\t'+aishell_scp[i]+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('aishell_2_trans.txt', 'w') as f:\n",
    "    for i in corp_map:\n",
    "        f.write(i+'\\t'+corp_map[i]+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 筛选一部分到aishell_2.scp.shuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !bash tools/random_line.sh aishell_2.scp 2 aishell_2.scp.shuff\n",
    "! cat aishell_2.scp > aishell_2.scp.shuff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 合并数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "SLR_wav_scp = {}\n",
    "SLR_wav_corp = {}\n",
    "with open(\"SLR_wav.scp.shuff\", \"r\") as f:\n",
    "    for line in f:\n",
    "        data = line.split()\n",
    "        SLR_wav_scp[data[0]] = data[1]\n",
    "        SLR_wav_corp[data[0]] = \"你好米雅\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "aishell_wav_scp = {}\n",
    "with open(\"aishell_2.scp.shuff\", \"r\") as f:\n",
    "    for line in f:\n",
    "        data = line.split()\n",
    "        aishell_wav_scp[data[0]] = data[1]\n",
    "        \n",
    "aishell_wav_corp = {}       \n",
    "with open(\"aishell_2_trans.txt\", \"r\") as f:\n",
    "    for line in f:\n",
    "        data = line.split()\n",
    "        if data[0] in aishell_wav_scp:\n",
    "            aishell_wav_corp[data[0]] = data[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "wav_scp = {**SLR_wav_scp,**aishell_wav_scp}\n",
    "corpus = {**SLR_wav_corp, **aishell_wav_corp}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('wav.scp', 'w') as f:\n",
    "    for i in sorted(wav_scp):\n",
    "        f.write(i+'\\t'+wav_scp[i]+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('corpus.txt', 'w') as f:\n",
    "    for i in sorted(corpus):\n",
    "        f.write(i+'\\t'+corpus[i]+'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 生成词典并分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "!sed 's/\\s.*$/ 1/' gmm/data/dict/lexicon.txt > seg.dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from /home1/meichaoyang/workspace/git/phone_align/seg.dict ...\n",
      "Dumping model to file cache /home1/meichaoyang/anaconda3/lib/python3.7/site-packages/jieba/cache/jieba.u9ddf4f6511f6ed6800c70d1cc27817bc.cache\n",
      "Loading model cost 0.445 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "#!python seg_word/segmentword.py seg.dict corpus.txt corpus.seg oov_file\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "vocab_file=\"seg.dict\"\n",
    "trans_file=\"corpus.txt\"\n",
    "word_segmented_trans=\"corpus.seg\"\n",
    "\n",
    "jieba.set_dictionary(vocab_file)\n",
    "with open(word_segmented_trans, \"w\") as f:\n",
    "    for line in open(trans_file):\n",
    "      key,trans = line.strip().split(None, 1)\n",
    "      words = jieba.cut(trans, HMM=False) # turn off new word discovery (HMM-based)\n",
    "      new_line = key + '\\t' + \" \".join(words)\n",
    "      f.write(new_line + \"\\n\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!python tools/filter_scp.py corpus.seg wav.scp wav.scp.new"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## utt2spk和spk2utt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_wav_scp_path = \"wav.scp\"\n",
    "new_wav_scp = {}\n",
    "with open(new_wav_scp_path, \"r\") as f:\n",
    "    for line in f:\n",
    "        data = line.split()\n",
    "        new_wav_scp[data[0]] = data[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'SV0255'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pattern = re.compile(u'/[^/]+')\n",
    "str1 = '/home/meichaoyang/dataset/SLR85/dev/SPEECHDATA/wav/SV0255/SV0255_2_00_F0026.wav'\n",
    "pattern.findall(str1)[-2][1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "895683"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new_wav_scp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "utts = list(new_wav_scp.items())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "utt2spk = {}\n",
    "for utt in utts:\n",
    "    spk = pattern.findall(utt[1])[-2][1:]\n",
    "    utt2spk[utt[0]] = spk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "spk2utt = {}\n",
    "for utt in utts:\n",
    "    spk = pattern.findall(utt[1])[-2][1:]\n",
    "    if spk not in spk2utt:\n",
    "        spk2utt[spk] = []\n",
    "    spk2utt[spk].append(utt[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('utt2spk', 'w') as f:\n",
    "    for i in sorted(utt2spk):\n",
    "        f.write(i+'\\t'+utt2spk[i]+'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "895683"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(utts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 根据utt2spk生成spk2utt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "!gmm/utils/utt2spk_to_spk2utt.pl utt2spk > spk2utt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### spk2utt，utt2spk，text(corpus.seg)，wav.scp(wav.scp.new)复制到train_{mfcc,fbank}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf train_mfcc train_fbank\n",
    "!mkdir train_mfcc train_fbank\n",
    "!cp spk2utt utt2spk corpus.seg wav.scp train_mfcc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mv train_mfcc/corpus.seg train_mfcc/text \n",
    "#!mv train_mfcc/wav.scp.new train_mfcc/wav.scp\n",
    "!cd train_mfcc &&cp spk2utt utt2spk text wav.scp ../train_fbank"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 特征提取`cd ext_feats && sh ext_{mfcc,fbank}.sh {train_fbank,train_mfcc,dev}`，需注意修改conf下mfcc，fbank参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "steps/make_mfcc.sh --nj 48 --cmd run.pl ../train_mfcc ../train_mfcc/log ../train_mfcc/_mfcc\n",
      "utils/validate_data_dir.sh: Successfully validated data-directory ../train_mfcc\n",
      "steps/make_mfcc.sh: [info]: no segments file exists: assuming wav.scp indexed by utterance.\n",
      "Succeeded creating MFCC features for train_mfcc\n",
      "steps/compute_cmvn_stats.sh ../train_mfcc ../train_mfcc/cmvn_log ../train_mfcc/_cmvn\n"
     ]
    }
   ],
   "source": [
    "!cd ext_feats && sh ext_mfcc.sh ../train_mfcc && utils/fix_data_dir.sh ../train_mfcc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Succeeded creating filterbank features for train_fbank\n",
      "steps/compute_cmvn_stats.sh ../train_fbank ../train_fbank/cmvn_log ../train_fbank/_cmvn\n",
      "Succeeded creating CMVN stats for train_fbank\n",
      "fix_data_dir.sh: kept all 895683 utterances.\n",
      "fix_data_dir.sh: old files are kept in ../train_fbank/.backup\n"
     ]
    }
   ],
   "source": [
    "!cd ext_feats && sh ext_fbank.sh ../train_fbank && utils/fix_data_dir.sh ../train_fbank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ln: failed to create symbolic link 'gmm/data/train/train_mfcc': File exists\n",
      "ln: failed to create symbolic link 'chain/data/train_mfcc/train': File exists\n"
     ]
    }
   ],
   "source": [
    "!ln -s ../../train_mfcc gmm/data/train\n",
    "!ln -s ../../train_fbank chain/data/train_fbank\n",
    "!ln -s ../../gmm/data/train/ chain/data/train_mfcc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "utils/prepare_lang.sh --position-dependent-phones false `pwd`/data/dict \"SPOKEN_NOISE\" `pwd`/data/local/lang `pwd`/data/lang\n",
      "utils/prepare_lang.sh --position-dependent-phones false /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict SPOKEN_NOISE /home1/meichaoyang/workspace/git/phone_align/gmm/data/local/lang /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/silence_phones.txt ...\n",
      "--> reading /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/silence_phones.txt\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/silence_phones.txt is OK\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/optional_silence.txt ...\n",
      "--> reading /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/optional_silence.txt\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/optional_silence.txt is OK\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/nonsilence_phones.txt ...\n",
      "--> reading /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/nonsilence_phones.txt\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/nonsilence_phones.txt is OK\n",
      "\n",
      "Checking disjoint: silence_phones.txt, nonsilence_phones.txt\n",
      "--> disjoint property is OK.\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/lexicon.txt\n",
      "--> reading /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/lexicon.txt\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/lexicon.txt is OK\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/extra_questions.txt ...\n",
      "--> reading /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/extra_questions.txt\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/extra_questions.txt is OK\n",
      "--> SUCCESS [validating dictionary directory /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict]\n",
      "\n",
      "**Creating /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/lexiconp.txt from /home1/meichaoyang/workspace/git/phone_align/gmm/data/dict/lexicon.txt\n",
      "fstaddselfloops /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/wdisambig_phones.int /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/wdisambig_words.int \n",
      "prepare_lang.sh: validating output directory\n",
      "utils/validate_lang.pl /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones.txt ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones.txt is OK\n",
      "\n",
      "Checking words.txt: #0 ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/words.txt is OK\n",
      "\n",
      "Checking disjoint: silence.txt, nonsilence.txt, disambig.txt ...\n",
      "--> silence.txt and nonsilence.txt are disjoint\n",
      "--> silence.txt and disambig.txt are disjoint\n",
      "--> disambig.txt and nonsilence.txt are disjoint\n",
      "--> disjoint property is OK\n",
      "\n",
      "Checking sumation: silence.txt, nonsilence.txt, disambig.txt ...\n",
      "--> found no unexplainable phones in phones.txt\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/context_indep.{txt, int, csl} ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> 2 entry/entries in /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/context_indep.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/context_indep.int corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/context_indep.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/context_indep.csl corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/context_indep.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/context_indep.{txt, int, csl} are OK\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/nonsilence.{txt, int, csl} ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> 217 entry/entries in /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/nonsilence.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/nonsilence.int corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/nonsilence.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/nonsilence.csl corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/nonsilence.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/nonsilence.{txt, int, csl} are OK\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/silence.{txt, int, csl} ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> 2 entry/entries in /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/silence.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/silence.int corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/silence.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/silence.csl corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/silence.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/silence.{txt, int, csl} are OK\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/optional_silence.{txt, int, csl} ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> 1 entry/entries in /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/optional_silence.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/optional_silence.int corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/optional_silence.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/optional_silence.csl corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/optional_silence.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/optional_silence.{txt, int, csl} are OK\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/disambig.{txt, int, csl} ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> 57 entry/entries in /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/disambig.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/disambig.int corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/disambig.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/disambig.csl corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/disambig.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/disambig.{txt, int, csl} are OK\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/roots.{txt, int} ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> 68 entry/entries in /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/roots.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/roots.int corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/roots.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/roots.{txt, int} are OK\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/sets.{txt, int} ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> 68 entry/entries in /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/sets.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/sets.int corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/sets.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/sets.{txt, int} are OK\n",
      "\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/extra_questions.{txt, int} ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> 7 entry/entries in /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/extra_questions.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/extra_questions.int corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/extra_questions.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/extra_questions.{txt, int} are OK\n",
      "\n",
      "Checking optional_silence.txt ...\n",
      "--> reading /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/optional_silence.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/optional_silence.txt is OK\n",
      "\n",
      "Checking disambiguation symbols: #0 and #1\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/disambig.txt has \"#0\" and \"#1\"\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/disambig.txt is OK\n",
      "\n",
      "Checking topo ...\n",
      "\n",
      "Checking word-level disambiguation symbols...\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/phones/wdisambig.txt exists (newer prepare_lang.sh)\n",
      "Checking /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/oov.{txt, int} ...\n",
      "--> text seems to be UTF-8 or ASCII, checking whitespaces\n",
      "--> text contains only allowed whitespaces\n",
      "--> 1 entry/entries in /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/oov.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/oov.int corresponds to /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/oov.txt\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/oov.{txt, int} are OK\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/L.fst is olabel sorted\n",
      "--> /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang/L_disambig.fst is olabel sorted\n",
      "--> SUCCESS [validating lang directory /home1/meichaoyang/workspace/git/phone_align/gmm/data/lang]\n"
     ]
    }
   ],
   "source": [
    "!cd gmm && make prepare_lang"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "local/train_gmm.sh `pwd`/data/train `pwd`/data/cv\n",
      "steps/train_mono.sh --boost-silence 1.25 --nj 24 --cmd run.pl /home1/meichaoyang/workspace/git/phone_align/gmm/data/train data/lang exp/mono\n",
      "steps/train_mono.sh: Initializing monophone system.\n",
      "steps/train_mono.sh: Compiling training graphs\n",
      "steps/train_mono.sh: Aligning data equally (pass 0)\n",
      "steps/train_mono.sh: Pass 1\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 2\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 3\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 4\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 5\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 6\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 7\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 8\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 9\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 10\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 11\n",
      "steps/train_mono.sh: Pass 12\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 13\n",
      "steps/train_mono.sh: Pass 14\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 15\n",
      "steps/train_mono.sh: Pass 16\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 17\n",
      "steps/train_mono.sh: Pass 18\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 19\n",
      "steps/train_mono.sh: Pass 20\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 21\n",
      "steps/train_mono.sh: Pass 22\n",
      "steps/train_mono.sh: Pass 23\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 24\n",
      "steps/train_mono.sh: Pass 25\n",
      "steps/train_mono.sh: Pass 26\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 27\n",
      "steps/train_mono.sh: Pass 28\n",
      "steps/train_mono.sh: Pass 29\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 30\n",
      "steps/train_mono.sh: Pass 31\n",
      "steps/train_mono.sh: Pass 32\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 33\n",
      "steps/train_mono.sh: Pass 34\n",
      "steps/train_mono.sh: Pass 35\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 36\n",
      "steps/train_mono.sh: Pass 37\n",
      "steps/train_mono.sh: Pass 38\n",
      "steps/train_mono.sh: Aligning data\n",
      "steps/train_mono.sh: Pass 39\n",
      "steps/diagnostic/analyze_alignments.sh --cmd run.pl data/lang exp/mono\n",
      "steps/diagnostic/analyze_alignments.sh: see stats in exp/mono/log/analyze_alignments.log\n",
      "6 warnings in exp/mono/log/init.log\n",
      "93578 warnings in exp/mono/log/align.*.*.log\n",
      "15719 warnings in exp/mono/log/acc.*.*.log\n",
      "exp/mono: nj=24 align prob=-93.56 over 870.86h [retry=0.2%, fail=0.1%] states=208 gauss=992\n",
      "steps/train_mono.sh: Done training monophone system in exp/mono\n",
      "steps/align_si.sh --boost-silence 1.25 --nj 24 --cmd run.pl /home1/meichaoyang/workspace/git/phone_align/gmm/data/train data/lang exp/mono exp/mono_ali\n",
      "steps/align_si.sh: feature type is delta\n",
      "steps/align_si.sh: aligning data in /home1/meichaoyang/workspace/git/phone_align/gmm/data/train using model from exp/mono, putting alignments in exp/mono_ali\n",
      "steps/diagnostic/analyze_alignments.sh --cmd run.pl data/lang exp/mono_ali\n",
      "steps/diagnostic/analyze_alignments.sh: see stats in exp/mono_ali/log/analyze_alignments.log\n",
      "steps/align_si.sh: done aligning data.\n",
      "steps/train_deltas.sh --boost-silence 1.25 --cmd run.pl 10000 80000 /home1/meichaoyang/workspace/git/phone_align/gmm/data/train data/lang exp/mono_ali exp/tri1\n",
      "steps/train_deltas.sh: accumulating tree stats\n",
      "steps/train_deltas.sh: getting questions for tree-building, via clustering\n",
      "steps/train_deltas.sh: building the tree\n",
      "steps/train_deltas.sh: converting alignments from exp/mono_ali to use current tree\n",
      "steps/train_deltas.sh: compiling graphs of transcripts\n",
      "steps/train_deltas.sh: training pass 1\n",
      "steps/train_deltas.sh: training pass 2\n",
      "steps/train_deltas.sh: training pass 3\n",
      "steps/train_deltas.sh: training pass 4\n",
      "steps/train_deltas.sh: training pass 5\n",
      "steps/train_deltas.sh: training pass 6\n",
      "steps/train_deltas.sh: training pass 7\n",
      "steps/train_deltas.sh: training pass 8\n",
      "steps/train_deltas.sh: training pass 9\n",
      "steps/train_deltas.sh: training pass 10\n",
      "steps/train_deltas.sh: aligning data\n",
      "steps/train_deltas.sh: training pass 11\n",
      "steps/train_deltas.sh: training pass 12\n",
      "steps/train_deltas.sh: training pass 13\n",
      "steps/train_deltas.sh: training pass 14\n",
      "steps/train_deltas.sh: training pass 15\n",
      "steps/train_deltas.sh: training pass 16\n",
      "steps/train_deltas.sh: training pass 17\n",
      "steps/train_deltas.sh: training pass 18\n",
      "steps/train_deltas.sh: training pass 19\n",
      "steps/train_deltas.sh: training pass 20\n",
      "steps/train_deltas.sh: aligning data\n",
      "steps/train_deltas.sh: training pass 21\n",
      "steps/train_deltas.sh: training pass 22\n",
      "steps/train_deltas.sh: training pass 23\n",
      "steps/train_deltas.sh: training pass 24\n",
      "steps/train_deltas.sh: training pass 25\n",
      "steps/train_deltas.sh: training pass 26\n",
      "steps/train_deltas.sh: training pass 27\n",
      "steps/train_deltas.sh: training pass 28\n",
      "steps/train_deltas.sh: training pass 29\n",
      "steps/train_deltas.sh: training pass 30\n",
      "steps/train_deltas.sh: aligning data\n",
      "steps/train_deltas.sh: training pass 31\n",
      "steps/train_deltas.sh: training pass 32\n",
      "steps/train_deltas.sh: training pass 33\n",
      "steps/train_deltas.sh: training pass 34\n",
      "steps/diagnostic/analyze_alignments.sh --cmd run.pl data/lang exp/tri1\n",
      "steps/diagnostic/analyze_alignments.sh: see stats in exp/tri1/log/analyze_alignments.log\n",
      "24550 warnings in exp/tri1/log/acc.*.*.log\n",
      "12158 warnings in exp/tri1/log/align.*.*.log\n",
      "1 warnings in exp/tri1/log/build_tree.log\n",
      "exp/tri1: nj=24 align prob=-91.18 over 870.31h [retry=0.4%, fail=0.1%] states=7680 gauss=80208 tree-impr=4.24\n",
      "steps/train_deltas.sh: Done training system with delta+delta-delta features in exp/tri1\n",
      "steps/align_si.sh --nj 24 --cmd run.pl /home1/meichaoyang/workspace/git/phone_align/gmm/data/train data/lang exp/tri1 exp/tri1_ali\n",
      "steps/align_si.sh: feature type is delta\n",
      "steps/align_si.sh: aligning data in /home1/meichaoyang/workspace/git/phone_align/gmm/data/train using model from exp/tri1, putting alignments in exp/tri1_ali\n",
      "steps/diagnostic/analyze_alignments.sh --cmd run.pl data/lang exp/tri1_ali\n",
      "steps/diagnostic/analyze_alignments.sh: see stats in exp/tri1_ali/log/analyze_alignments.log\n",
      "steps/align_si.sh: done aligning data.\n",
      "steps/train_lda_mllt.sh --cmd run.pl --splice-opts --left-context=3 --right-context=3 12000 120000 /home1/meichaoyang/workspace/git/phone_align/gmm/data/train data/lang exp/tri1_ali exp/tri2b\n",
      "steps/train_lda_mllt.sh: Accumulating LDA statistics.\n",
      "steps/train_lda_mllt.sh: Accumulating tree stats\n",
      "steps/train_lda_mllt.sh: Getting questions for tree clustering.\n",
      "steps/train_lda_mllt.sh: Building the tree\n",
      "steps/train_lda_mllt.sh: Initializing the model\n",
      "steps/train_lda_mllt.sh: Converting alignments from exp/tri1_ali to use current tree\n",
      "steps/train_lda_mllt.sh: Compiling graphs of transcripts\n",
      "Training pass 1\n",
      "Training pass 2\n",
      "steps/train_lda_mllt.sh: Estimating MLLT\n",
      "Training pass 3\n",
      "Training pass 4\n",
      "steps/train_lda_mllt.sh: Estimating MLLT\n",
      "Training pass 5\n",
      "Training pass 6\n",
      "steps/train_lda_mllt.sh: Estimating MLLT\n",
      "Training pass 7\n",
      "Training pass 8\n",
      "Training pass 9\n",
      "Training pass 10\n",
      "Aligning data\n",
      "Training pass 11\n",
      "Training pass 12\n",
      "steps/train_lda_mllt.sh: Estimating MLLT\n",
      "Training pass 13\n",
      "Training pass 14\n",
      "Training pass 15\n",
      "Training pass 16\n",
      "Training pass 17\n",
      "Training pass 18\n",
      "Training pass 19\n",
      "Training pass 20\n",
      "Aligning data\n",
      "Training pass 21\n",
      "Training pass 22\n",
      "Training pass 23\n",
      "Training pass 24\n",
      "Training pass 25\n",
      "Training pass 26\n",
      "Training pass 27\n",
      "Training pass 28\n",
      "Training pass 29\n",
      "Training pass 30\n",
      "Aligning data\n",
      "Training pass 31\n",
      "Training pass 32\n",
      "Training pass 33\n",
      "Training pass 34\n",
      "steps/diagnostic/analyze_alignments.sh --cmd run.pl data/lang exp/tri2b\n",
      "steps/diagnostic/analyze_alignments.sh: see stats in exp/tri2b/log/analyze_alignments.log\n",
      "1 warnings in exp/tri2b/log/build_tree.log\n",
      "15224 warnings in exp/tri2b/log/align.*.*.log\n",
      "937 warnings in exp/tri2b/log/lda_acc.*.log\n",
      "1 warnings in exp/tri2b/log/questions.log\n",
      "34168 warnings in exp/tri2b/log/acc.*.*.log\n",
      "exp/tri2b: nj=24 align prob=-48.15 over 870.15h [retry=0.4%, fail=0.1%] states=9576 gauss=120246 tree-impr=4.61 lda-sum=21.12 mllt:impr,logdet=0.91,1.40\n",
      "steps/train_lda_mllt.sh: Done training system with LDA+MLLT features in exp/tri2b\n",
      "steps/align_si.sh --nj 24 --cmd run.pl --use-graphs true /home1/meichaoyang/workspace/git/phone_align/gmm/data/train data/lang exp/tri2b exp/tri2b_ali\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "steps/align_si.sh: feature type is lda\n",
      "steps/align_si.sh: aligning data in /home1/meichaoyang/workspace/git/phone_align/gmm/data/train using model from exp/tri2b, putting alignments in exp/tri2b_ali\n",
      "steps/diagnostic/analyze_alignments.sh --cmd run.pl data/lang exp/tri2b_ali\n",
      "steps/diagnostic/analyze_alignments.sh: see stats in exp/tri2b_ali/log/analyze_alignments.log\n",
      "steps/align_si.sh: done aligning data.\n",
      "steps/train_sat.sh --cmd run.pl 15000 180000 /home1/meichaoyang/workspace/git/phone_align/gmm/data/train data/lang exp/tri2b_ali exp/tri3b\n",
      "steps/train_sat.sh: feature type is lda\n",
      "steps/train_sat.sh: obtaining initial fMLLR transforms since not present in exp/tri2b_ali\n",
      "steps/train_sat.sh: Accumulating tree stats\n",
      "steps/train_sat.sh: Getting questions for tree clustering.\n",
      "steps/train_sat.sh: Building the tree\n",
      "steps/train_sat.sh: Initializing the model\n",
      "steps/train_sat.sh: Converting alignments from exp/tri2b_ali to use current tree\n",
      "steps/train_sat.sh: Compiling graphs of transcripts\n",
      "Pass 1\n",
      "Pass 2\n",
      "Estimating fMLLR transforms\n",
      "Pass 3\n",
      "Pass 4\n",
      "Estimating fMLLR transforms\n",
      "Pass 5\n",
      "Pass 6\n",
      "Estimating fMLLR transforms\n",
      "Pass 7\n",
      "Pass 8\n",
      "Pass 9\n",
      "Pass 10\n",
      "Aligning data\n",
      "Pass 11\n",
      "Pass 12\n",
      "Estimating fMLLR transforms\n",
      "Pass 13\n",
      "Pass 14\n",
      "Pass 15\n",
      "Pass 16\n",
      "Pass 17\n",
      "Pass 18\n",
      "Pass 19\n",
      "Pass 20\n",
      "Aligning data\n",
      "Pass 21\n",
      "Pass 22\n",
      "Pass 23\n",
      "Pass 24\n",
      "Pass 25\n",
      "Pass 26\n",
      "Pass 27\n",
      "Pass 28\n",
      "Pass 29\n",
      "Pass 30\n",
      "Aligning data\n",
      "Pass 31\n",
      "Pass 32\n",
      "Pass 33\n",
      "Pass 34\n",
      "steps/diagnostic/analyze_alignments.sh --cmd run.pl data/lang exp/tri3b\n",
      "steps/diagnostic/analyze_alignments.sh: see stats in exp/tri3b/log/analyze_alignments.log\n",
      "15709 warnings in exp/tri3b/log/align.*.*.log\n",
      "37549 warnings in exp/tri3b/log/acc.*.*.log\n",
      "5404 warnings in exp/tri3b/log/fmllr.*.*.log\n",
      "1 warnings in exp/tri3b/log/build_tree.log\n",
      "1 warnings in exp/tri3b/log/questions.log\n",
      "steps/train_sat.sh: Likelihood evolution:\n",
      "-47.9419 -47.7009 -47.4755 -47.4086 -47.0192 -46.5516 -46.2012 -45.9594 -45.7708 -45.3584 -45.2196 -44.9799 -44.8884 -44.8228 -44.7622 -44.7064 -44.6563 -44.6115 -44.5708 -44.471 -44.4188 -44.3862 -44.3566 -44.3294 -44.3043 -44.2807 -44.2582 -44.2365 -44.2157 -44.1654 -44.1346 -44.117 -44.1037 -44.0938 \n",
      "exp/tri3b: nj=24 align prob=-47.33 over 870.14h [retry=0.4%, fail=0.1%] states=12128 gauss=180218 fmllr-impr=3.24 over 659.93h tree-impr=6.44\n",
      "steps/train_sat.sh: done training SAT system in exp/tri3b\n",
      "steps/align_fmllr.sh --nj 24 --cmd run.pl /home1/meichaoyang/workspace/git/phone_align/gmm/data/train data/lang exp/tri3b exp/tri3b_ali\n",
      "steps/align_fmllr.sh: feature type is lda\n",
      "steps/align_fmllr.sh: compiling training graphs\n",
      "steps/align_fmllr.sh: aligning data in /home1/meichaoyang/workspace/git/phone_align/gmm/data/train using exp/tri3b/final.alimdl and speaker-independent features.\n",
      "steps/align_fmllr.sh: computing fMLLR transforms\n",
      "steps/align_fmllr.sh: doing final alignment.\n",
      "steps/align_fmllr.sh: done aligning data.\n",
      "steps/diagnostic/analyze_alignments.sh --cmd run.pl data/lang exp/tri3b_ali\n",
      "steps/diagnostic/analyze_alignments.sh: see stats in exp/tri3b_ali/log/analyze_alignments.log\n",
      "896 warnings in exp/tri3b_ali/log/fmllr.*.log\n",
      "3942 warnings in exp/tri3b_ali/log/align_pass1.*.log\n",
      "4784 warnings in exp/tri3b_ali/log/align_pass2.*.log\n",
      "steps/train_quick.sh --cmd run.pl 18000 250000 /home1/meichaoyang/workspace/git/phone_align/gmm/data/train data/lang exp/tri3b_ali exp/tri4b\n",
      "steps/train_quick.sh: feature type is lda\n",
      "steps/train_quick.sh: using transforms from exp/tri3b_ali\n",
      "steps/train_quick.sh: accumulating tree stats\n",
      "steps/train_quick.sh: Getting questions for tree clustering.\n",
      "steps/train_quick.sh: Building the tree\n",
      "steps/train_quick.sh: Initializing the model\n",
      "steps/train_quick.sh: mixing up old model.\n",
      "steps/train_quick.sh: converting old alignments\n",
      "steps/train_quick.sh: compiling training graphs\n",
      "steps/train_quick.sh: pass 1\n",
      "steps/train_quick.sh: pass 2\n",
      "steps/train_quick.sh: pass 3\n",
      "steps/train_quick.sh: pass 4\n",
      "steps/train_quick.sh: pass 5\n",
      "steps/train_quick.sh: pass 6\n",
      "steps/train_quick.sh: pass 7\n",
      "steps/train_quick.sh: pass 8\n",
      "steps/train_quick.sh: pass 9\n",
      "steps/train_quick.sh: pass 10\n",
      "steps/train_quick.sh: aligning data\n",
      "steps/train_quick.sh: pass 11\n",
      "steps/train_quick.sh: pass 12\n",
      "steps/train_quick.sh: pass 13\n",
      "steps/train_quick.sh: pass 14\n",
      "steps/train_quick.sh: pass 15\n",
      "steps/train_quick.sh: aligning data\n",
      "steps/train_quick.sh: pass 16\n",
      "steps/train_quick.sh: pass 17\n",
      "steps/train_quick.sh: pass 18\n",
      "steps/train_quick.sh: pass 19\n",
      "steps/train_quick.sh: estimating alignment model\n",
      "Done\n",
      "steps/align_fmllr.sh --nj 24 --cmd run.pl /home1/meichaoyang/workspace/git/phone_align/gmm/data/train data/lang exp/tri4b exp/tri4b_ali\n",
      "steps/align_fmllr.sh: feature type is lda\n",
      "steps/align_fmllr.sh: compiling training graphs\n",
      "steps/align_fmllr.sh: aligning data in /home1/meichaoyang/workspace/git/phone_align/gmm/data/train using exp/tri4b/final.alimdl and speaker-independent features.\n",
      "steps/align_fmllr.sh: computing fMLLR transforms\n",
      "steps/align_fmllr.sh: doing final alignment.\n",
      "steps/align_fmllr.sh: done aligning data.\n",
      "steps/diagnostic/analyze_alignments.sh --cmd run.pl data/lang exp/tri4b_ali\n",
      "steps/diagnostic/analyze_alignments.sh: see stats in exp/tri4b_ali/log/analyze_alignments.log\n",
      "894 warnings in exp/tri4b_ali/log/fmllr.*.log\n",
      "3769 warnings in exp/tri4b_ali/log/align_pass1.*.log\n",
      "4697 warnings in exp/tri4b_ali/log/align_pass2.*.log\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "!cd gmm &&make -f Makefile gmm  | tee gmm.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "local/train_chain.sh exp/chain/tdnn_attend\n",
      "local/train_chain.sh exp/chain/tdnn_attend\n",
      "steps/align_fmllr_lats.sh --nj 24 --cmd run.pl data/train_mfcc data/lang exp/tri4b exp/tri4b_lats\n",
      "steps/align_fmllr_lats.sh: feature type is lda\n",
      "steps/align_fmllr_lats.sh: compiling training graphs\n",
      "steps/align_fmllr_lats.sh: aligning data in data/train_mfcc using exp/tri4b/final.alimdl and speaker-independent features.\n",
      "steps/align_fmllr_lats.sh: computing fMLLR transforms\n",
      "steps/align_fmllr_lats.sh: generating lattices containing alternate pronunciations.\n",
      "steps/align_fmllr_lats.sh: done generating lattices from training transcripts.\n",
      "894 warnings in exp/tri4b_lats/log/fmllr.*.log\n",
      "691 warnings in exp/tri4b_lats/log/generate_lattices.*.log\n",
      "3809 warnings in exp/tri4b_lats/log/align_pass1.*.log\n",
      "steps/nnet3/chain/build_tree.sh --frame-subsampling-factor 3 --leftmost-questions-truncate -1 --context-opts --context-width=2 --central-position=1 --cmd run.pl 7000 data/train_mfcc data/lang_new exp/tri4b_ali exp/chain/tri4_tree\n",
      "steps/nnet3/chain/build_tree.sh: feature type is lda\n",
      "steps/nnet3/chain/build_tree.sh: Using transforms from exp/tri4b_ali\n",
      "steps/nnet3/chain/build_tree.sh: Initializing monophone model (for alignment conversion, in case topology changed)\n",
      "steps/nnet3/chain/build_tree.sh: Accumulating tree stats\n",
      "steps/nnet3/chain/build_tree.sh: Getting questions for tree clustering.\n",
      "steps/nnet3/chain/build_tree.sh: Building the tree\n",
      "steps/nnet3/chain/build_tree.sh: Initializing the model\n",
      "steps/nnet3/chain/build_tree.sh: Converting alignments from exp/tri4b_ali to use current tree\n",
      "steps/nnet3/chain/build_tree.sh: Done building tree\n",
      "local/train_chain.sh: creating neural net configs using the xconfig parser\n",
      "tree-info exp/chain/tri4_tree/tree \n",
      "steps/nnet3/xconfig_to_configs.py --xconfig-file exp/chain/tdnn_attend/configs/network.xconfig --config-dir exp/chain/tdnn_attend/configs/\n",
      "nnet3-init exp/chain/tdnn_attend/configs//init.config exp/chain/tdnn_attend/configs//init.raw \n",
      "LOG (nnet3-init[5.5]:main():nnet3-init.cc:80) Initialized raw neural net and wrote it to exp/chain/tdnn_attend/configs//init.raw\n",
      "nnet3-info exp/chain/tdnn_attend/configs//init.raw \n",
      "nnet3-init exp/chain/tdnn_attend/configs//ref.config exp/chain/tdnn_attend/configs//ref.raw \n",
      "LOG (nnet3-init[5.5]:main():nnet3-init.cc:80) Initialized raw neural net and wrote it to exp/chain/tdnn_attend/configs//ref.raw\n",
      "nnet3-info exp/chain/tdnn_attend/configs//ref.raw \n",
      "nnet3-init exp/chain/tdnn_attend/configs//ref.config exp/chain/tdnn_attend/configs//ref.raw \n",
      "LOG (nnet3-init[5.5]:main():nnet3-init.cc:80) Initialized raw neural net and wrote it to exp/chain/tdnn_attend/configs//ref.raw\n",
      "nnet3-info exp/chain/tdnn_attend/configs//ref.raw \n",
      "2020-03-18 11:41:32,583 [steps/nnet3/chain/train.py:35 - <module> - INFO ] Starting chain model trainer (train.py)\n",
      "2020-03-18 11:41:32,591 [steps/nnet3/chain/train.py:273 - train - INFO ] Arguments for the experiment\n",
      "{'alignment_subsampling_factor': 3,\n",
      " 'apply_deriv_weights': False,\n",
      " 'backstitch_training_interval': 1,\n",
      " 'backstitch_training_scale': 0.0,\n",
      " 'chunk_left_context': 0,\n",
      " 'chunk_left_context_initial': -1,\n",
      " 'chunk_right_context': 0,\n",
      " 'chunk_right_context_final': -1,\n",
      " 'chunk_width': '150',\n",
      " 'cleanup': True,\n",
      " 'cmvn_opts': '--norm-means=false --norm-vars=false',\n",
      " 'combine_sum_to_one_penalty': 0.0,\n",
      " 'command': 'run.pl --mem 32G',\n",
      " 'compute_per_dim_accuracy': False,\n",
      " 'deriv_truncate_margin': None,\n",
      " 'dir': 'exp/chain/tdnn_attend',\n",
      " 'do_final_combination': True,\n",
      " 'dropout_schedule': None,\n",
      " 'egs_command': None,\n",
      " 'egs_dir': None,\n",
      " 'egs_opts': '--frames-overlap-per-eg 0',\n",
      " 'egs_stage': -10,\n",
      " 'email': None,\n",
      " 'exit_stage': None,\n",
      " 'feat_dir': 'data/train_fbank',\n",
      " 'final_effective_lrate': 0.0001,\n",
      " 'frame_subsampling_factor': 3,\n",
      " 'frames_per_iter': 1500000,\n",
      " 'initial_effective_lrate': 0.001,\n",
      " 'input_model': None,\n",
      " 'l2_regularize': 5e-05,\n",
      " 'lat_dir': 'exp/tri4b_lats',\n",
      " 'leaky_hmm_coefficient': 0.1,\n",
      " 'left_deriv_truncate': None,\n",
      " 'left_tolerance': 5,\n",
      " 'lm_opts': '--num-extra-lm-states=2000',\n",
      " 'max_lda_jobs': 10,\n",
      " 'max_models_combine': 20,\n",
      " 'max_objective_evaluations': 30,\n",
      " 'max_param_change': 2.0,\n",
      " 'momentum': 0.0,\n",
      " 'num_chunk_per_minibatch': '128',\n",
      " 'num_epochs': 4.0,\n",
      " 'num_jobs_final': 3,\n",
      " 'num_jobs_initial': 1,\n",
      " 'online_ivector_dir': None,\n",
      " 'preserve_model_interval': 100,\n",
      " 'presoftmax_prior_scale_power': -0.25,\n",
      " 'proportional_shrink': 0.0,\n",
      " 'rand_prune': 4.0,\n",
      " 'remove_egs': False,\n",
      " 'reporting_interval': 0.1,\n",
      " 'right_tolerance': 5,\n",
      " 'samples_per_iter': 400000,\n",
      " 'shrink_saturation_threshold': 0.4,\n",
      " 'shrink_value': 1.0,\n",
      " 'shuffle_buffer_size': 5000,\n",
      " 'srand': 0,\n",
      " 'stage': -10,\n",
      " 'train_opts': [],\n",
      " 'tree_dir': 'exp/chain/tri4_tree',\n",
      " 'use_gpu': 'yes',\n",
      " 'xent_regularize': 0.1}\n",
      "2020-03-18 11:41:55,895 [steps/nnet3/chain/train.py:327 - train - INFO ] Creating phone language-model\n",
      "2020-03-18 11:42:24,652 [steps/nnet3/chain/train.py:332 - train - INFO ] Creating denominator FST\n",
      "copy-transition-model exp/chain/tri4_tree/final.mdl exp/chain/tdnn_attend/0.trans_mdl \n",
      "LOG (copy-transition-model[5.5]:main():copy-transition-model.cc:62) Copied transition model.\n",
      "2020-03-18 11:42:28,185 [steps/nnet3/chain/train.py:339 - train - INFO ] Initializing a basic network for estimating preconditioning matrix\n",
      "2020-03-18 11:42:28,237 [steps/nnet3/chain/train.py:361 - train - INFO ] Generating egs\n",
      "steps/nnet3/chain/get_egs.sh --frames-overlap-per-eg 0 --cmd run.pl --mem 32G --cmvn-opts --norm-means=false --norm-vars=false --online-ivector-dir  --left-context 28 --right-context 19 --left-context-initial -1 --right-context-final -1 --left-tolerance 5 --right-tolerance 5 --frame-subsampling-factor 3 --alignment-subsampling-factor 3 --stage -10 --frames-per-iter 1500000 --frames-per-eg 150 --srand 0 data/train_fbank exp/chain/tdnn_attend exp/tri4b_lats exp/chain/tdnn_attend/egs\n",
      "utils/data/get_frame_shift.sh: data/train_fbank/utt2dur does not exist: creating it\n",
      "utils/data/get_utt2dur.sh: segments file does not exist so getting durations from wave files\n",
      "cat: write error: Broken pipe\n",
      "utils/data/get_utt2dur.sh: could not get utterance lengths from sphere-file headers, using wav-to-duration\n",
      "utils/data/get_utt2dur.sh: computed data/train_fbank/utt2dur\n",
      "feat-to-len 'scp:head -n 10 data/train_fbank/feats.scp|' ark,t:- \n",
      "steps/nnet3/chain/get_egs.sh: creating egs.  To ensure they are not deleted later you can do:  touch exp/chain/tdnn_attend/egs/.nodelete\n",
      "steps/nnet3/chain/get_egs.sh: feature type is raw\n",
      "tree-info exp/chain/tdnn_attend/tree \n",
      "steps/nnet3/chain/get_egs.sh: working out number of frames of training data\n",
      "steps/nnet3/chain/get_egs.sh: working out feature dim\n",
      "steps/nnet3/chain/get_egs.sh: creating 211 archives, each with 9966 egs, with\n",
      "steps/nnet3/chain/get_egs.sh:   150 labels per example, and (left,right) context = (28,19)\n",
      "steps/nnet3/chain/get_egs.sh: Getting validation and training subset examples in background.\n",
      "steps/nnet3/chain/get_egs.sh: Generating training examples on disk\n",
      "... Getting subsets of validation examples for diagnostics and combination.\n",
      "steps/nnet3/chain/get_egs.sh: recombining and shuffling order of archives on disk\n",
      "steps/nnet3/chain/get_egs.sh: removing temporary archives\n",
      "steps/nnet3/chain/get_egs.sh: removing temporary alignments, lattices and transforms\n",
      "steps/nnet3/chain/get_egs.sh: Finished preparing training examples\n",
      "2020-03-18 11:52:36,913 [steps/nnet3/chain/train.py:410 - train - INFO ] Copying the properties from exp/chain/tdnn_attend/egs to exp/chain/tdnn_attend\n",
      "2020-03-18 11:52:36,914 [steps/nnet3/chain/train.py:424 - train - INFO ] Computing the preconditioning matrix for input features\n",
      "2020-03-18 11:52:59,584 [steps/nnet3/chain/train.py:433 - train - INFO ] Preparing the initial acoustic model.\n",
      "2020-03-18 11:53:00,764 [steps/nnet3/chain/train.py:467 - train - INFO ] Training will run for 4.0 epochs = 1266 iterations\n",
      "2020-03-18 11:53:00,765 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 0/1265    Epoch: 0.00/4.0 (0.0% complete)    lr: 0.001000    \n",
      "2020-03-18 11:53:51,339 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 1/1265    Epoch: 0.00/4.0 (0.0% complete)    lr: 0.000999    \n",
      "2020-03-18 11:54:21,769 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 2/1265    Epoch: 0.00/4.0 (0.1% complete)    lr: 0.000998    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 11:54:52,690 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 3/1265    Epoch: 0.00/4.0 (0.1% complete)    lr: 0.000997    \n",
      "2020-03-18 11:55:23,756 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 4/1265    Epoch: 0.01/4.0 (0.2% complete)    lr: 0.000996    \n",
      "2020-03-18 11:55:54,593 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 5/1265    Epoch: 0.01/4.0 (0.2% complete)    lr: 0.000995    \n",
      "2020-03-18 11:56:25,694 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 6/1265    Epoch: 0.01/4.0 (0.2% complete)    lr: 0.000995    \n",
      "2020-03-18 11:56:57,040 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 7/1265    Epoch: 0.01/4.0 (0.3% complete)    lr: 0.000994    \n",
      "2020-03-18 11:57:31,454 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 8/1265    Epoch: 0.01/4.0 (0.3% complete)    lr: 0.000993    \n",
      "2020-03-18 11:58:06,263 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 9/1265    Epoch: 0.01/4.0 (0.4% complete)    lr: 0.000992    \n",
      "2020-03-18 11:58:40,614 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 10/1265    Epoch: 0.02/4.0 (0.4% complete)    lr: 0.000991    \n",
      "2020-03-18 11:59:15,209 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 11/1265    Epoch: 0.02/4.0 (0.4% complete)    lr: 0.000990    \n",
      "2020-03-18 11:59:49,730 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 12/1265    Epoch: 0.02/4.0 (0.5% complete)    lr: 0.000989    \n",
      "2020-03-18 12:00:25,472 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 13/1265    Epoch: 0.02/4.0 (0.5% complete)    lr: 0.000988    \n",
      "2020-03-18 12:01:00,969 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 14/1265    Epoch: 0.02/4.0 (0.6% complete)    lr: 0.000987    \n",
      "2020-03-18 12:01:36,194 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 15/1265    Epoch: 0.02/4.0 (0.6% complete)    lr: 0.000986    \n",
      "2020-03-18 12:02:10,796 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 16/1265    Epoch: 0.03/4.0 (0.6% complete)    lr: 0.000986    \n",
      "2020-03-18 12:02:45,570 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 17/1265    Epoch: 0.03/4.0 (0.7% complete)    lr: 0.000985    \n",
      "2020-03-18 12:03:19,745 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 18/1265    Epoch: 0.03/4.0 (0.7% complete)    lr: 0.000984    \n",
      "2020-03-18 12:03:54,776 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 19/1265    Epoch: 0.03/4.0 (0.8% complete)    lr: 0.000983    \n",
      "2020-03-18 12:04:30,035 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 20/1265    Epoch: 0.03/4.0 (0.8% complete)    lr: 0.000982    \n",
      "2020-03-18 12:05:06,246 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 21/1265    Epoch: 0.03/4.0 (0.8% complete)    lr: 0.000981    \n",
      "2020-03-18 12:05:40,764 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 22/1265    Epoch: 0.03/4.0 (0.9% complete)    lr: 0.000980    \n",
      "2020-03-18 12:06:14,835 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 23/1265    Epoch: 0.04/4.0 (0.9% complete)    lr: 0.000979    \n",
      "2020-03-18 12:06:49,933 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 24/1265    Epoch: 0.04/4.0 (0.9% complete)    lr: 0.000978    \n",
      "2020-03-18 12:07:24,912 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 25/1265    Epoch: 0.04/4.0 (1.0% complete)    lr: 0.000978    \n",
      "2020-03-18 12:07:59,341 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 26/1265    Epoch: 0.04/4.0 (1.0% complete)    lr: 0.000977    \n",
      "2020-03-18 12:08:33,218 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 27/1265    Epoch: 0.04/4.0 (1.1% complete)    lr: 0.000976    \n",
      "2020-03-18 12:09:08,875 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 28/1265    Epoch: 0.04/4.0 (1.1% complete)    lr: 0.000975    \n",
      "2020-03-18 12:09:43,124 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 29/1265    Epoch: 0.05/4.0 (1.1% complete)    lr: 0.000974    \n",
      "2020-03-18 12:10:17,857 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 30/1265    Epoch: 0.05/4.0 (1.2% complete)    lr: 0.000973    \n",
      "2020-03-18 12:10:52,329 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 31/1265    Epoch: 0.05/4.0 (1.2% complete)    lr: 0.000972    \n",
      "2020-03-18 12:11:26,923 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 32/1265    Epoch: 0.05/4.0 (1.3% complete)    lr: 0.000971    \n",
      "2020-03-18 12:12:01,666 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 33/1265    Epoch: 0.05/4.0 (1.3% complete)    lr: 0.000970    \n",
      "2020-03-18 12:12:36,132 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 34/1265    Epoch: 0.05/4.0 (1.3% complete)    lr: 0.000970    \n",
      "2020-03-18 12:13:10,736 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 35/1265    Epoch: 0.06/4.0 (1.4% complete)    lr: 0.000969    \n",
      "2020-03-18 12:13:44,733 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 36/1265    Epoch: 0.06/4.0 (1.4% complete)    lr: 0.000968    \n",
      "2020-03-18 12:14:18,805 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 37/1265    Epoch: 0.06/4.0 (1.5% complete)    lr: 0.000967    \n",
      "2020-03-18 12:14:53,198 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 38/1265    Epoch: 0.06/4.0 (1.5% complete)    lr: 0.000966    \n",
      "2020-03-18 12:15:27,838 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 39/1265    Epoch: 0.06/4.0 (1.5% complete)    lr: 0.000965    \n",
      "2020-03-18 12:16:02,196 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 40/1265    Epoch: 0.06/4.0 (1.6% complete)    lr: 0.000964    \n",
      "2020-03-18 12:16:37,635 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 41/1265    Epoch: 0.06/4.0 (1.6% complete)    lr: 0.000963    \n",
      "2020-03-18 12:17:11,717 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 42/1265    Epoch: 0.07/4.0 (1.7% complete)    lr: 0.000963    \n",
      "2020-03-18 12:17:46,305 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 43/1265    Epoch: 0.07/4.0 (1.7% complete)    lr: 0.000962    \n",
      "2020-03-18 12:18:20,590 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 44/1265    Epoch: 0.07/4.0 (1.7% complete)    lr: 0.000961    \n",
      "2020-03-18 12:18:54,207 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 45/1265    Epoch: 0.07/4.0 (1.8% complete)    lr: 0.000960    \n",
      "2020-03-18 12:19:28,859 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 46/1265    Epoch: 0.07/4.0 (1.8% complete)    lr: 0.000959    \n",
      "2020-03-18 12:20:03,239 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 47/1265    Epoch: 0.07/4.0 (1.9% complete)    lr: 0.000958    \n",
      "2020-03-18 12:20:37,873 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 48/1265    Epoch: 0.08/4.0 (1.9% complete)    lr: 0.000957    \n",
      "2020-03-18 12:21:12,237 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 49/1265    Epoch: 0.08/4.0 (1.9% complete)    lr: 0.000956    \n",
      "2020-03-18 12:21:47,127 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 50/1265    Epoch: 0.08/4.0 (2.0% complete)    lr: 0.000956    \n",
      "2020-03-18 12:22:21,338 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 51/1265    Epoch: 0.08/4.0 (2.0% complete)    lr: 0.000955    \n",
      "2020-03-18 12:22:55,491 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 52/1265    Epoch: 0.08/4.0 (2.1% complete)    lr: 0.000954    \n",
      "2020-03-18 12:23:30,234 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 53/1265    Epoch: 0.08/4.0 (2.1% complete)    lr: 0.000953    \n",
      "2020-03-18 12:24:04,966 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 54/1265    Epoch: 0.09/4.0 (2.1% complete)    lr: 0.000952    \n",
      "2020-03-18 12:24:40,026 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 55/1265    Epoch: 0.09/4.0 (2.2% complete)    lr: 0.000951    \n",
      "2020-03-18 12:25:13,867 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 56/1265    Epoch: 0.09/4.0 (2.2% complete)    lr: 0.000950    \n",
      "2020-03-18 12:25:48,222 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 57/1265    Epoch: 0.09/4.0 (2.3% complete)    lr: 0.000949    \n",
      "2020-03-18 12:26:22,981 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 58/1265    Epoch: 0.09/4.0 (2.3% complete)    lr: 0.000949    \n",
      "2020-03-18 12:26:56,999 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 59/1265    Epoch: 0.09/4.0 (2.3% complete)    lr: 0.000948    \n",
      "2020-03-18 12:27:31,656 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 60/1265    Epoch: 0.09/4.0 (2.4% complete)    lr: 0.000947    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 12:28:07,390 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 61/1265    Epoch: 0.10/4.0 (2.4% complete)    lr: 0.000946    \n",
      "2020-03-18 12:28:41,985 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 62/1265    Epoch: 0.10/4.0 (2.4% complete)    lr: 0.000945    \n",
      "2020-03-18 12:29:16,663 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 63/1265    Epoch: 0.10/4.0 (2.5% complete)    lr: 0.000944    \n",
      "2020-03-18 12:29:50,578 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 64/1265    Epoch: 0.10/4.0 (2.5% complete)    lr: 0.000943    \n",
      "2020-03-18 12:30:24,741 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 65/1265    Epoch: 0.10/4.0 (2.6% complete)    lr: 0.000943    \n",
      "2020-03-18 12:30:59,465 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 66/1265    Epoch: 0.10/4.0 (2.6% complete)    lr: 0.000942    \n",
      "2020-03-18 12:31:32,764 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 67/1265    Epoch: 0.11/4.0 (2.6% complete)    lr: 0.000941    \n",
      "2020-03-18 12:32:07,018 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 68/1265    Epoch: 0.11/4.0 (2.7% complete)    lr: 0.000940    \n",
      "2020-03-18 12:32:41,718 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 69/1265    Epoch: 0.11/4.0 (2.7% complete)    lr: 0.000939    \n",
      "2020-03-18 12:33:14,574 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 70/1265    Epoch: 0.11/4.0 (2.8% complete)    lr: 0.000938    \n",
      "2020-03-18 12:33:49,298 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 71/1265    Epoch: 0.11/4.0 (2.8% complete)    lr: 0.000937    \n",
      "2020-03-18 12:34:25,116 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 72/1265    Epoch: 0.11/4.0 (2.8% complete)    lr: 0.000937    \n",
      "2020-03-18 12:35:00,090 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 73/1265    Epoch: 0.12/4.0 (2.9% complete)    lr: 0.000936    \n",
      "2020-03-18 12:35:33,814 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 74/1265    Epoch: 0.12/4.0 (2.9% complete)    lr: 0.000935    \n",
      "2020-03-18 12:36:08,478 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 75/1265    Epoch: 0.12/4.0 (3.0% complete)    lr: 0.000934    \n",
      "2020-03-18 12:36:43,389 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 76/1265    Epoch: 0.12/4.0 (3.0% complete)    lr: 0.000933    \n",
      "2020-03-18 12:37:17,388 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 77/1265    Epoch: 0.12/4.0 (3.0% complete)    lr: 0.000932    \n",
      "2020-03-18 12:37:51,118 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 78/1265    Epoch: 0.12/4.0 (3.1% complete)    lr: 0.000932    \n",
      "2020-03-18 12:38:25,235 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 79/1265    Epoch: 0.12/4.0 (3.1% complete)    lr: 0.000931    \n",
      "2020-03-18 12:38:59,665 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 80/1265    Epoch: 0.13/4.0 (3.2% complete)    lr: 0.000930    \n",
      "2020-03-18 12:39:35,019 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 81/1265    Epoch: 0.13/4.0 (3.2% complete)    lr: 0.000929    \n",
      "2020-03-18 12:40:09,325 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 82/1265    Epoch: 0.13/4.0 (3.2% complete)    lr: 0.000928    \n",
      "2020-03-18 12:40:44,107 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 83/1265    Epoch: 0.13/4.0 (3.3% complete)    lr: 0.000927    \n",
      "2020-03-18 12:41:18,882 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 84/1265    Epoch: 0.13/4.0 (3.3% complete)    lr: 0.000926    \n",
      "2020-03-18 12:41:54,559 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 85/1265    Epoch: 0.13/4.0 (3.4% complete)    lr: 0.000926    \n",
      "2020-03-18 12:42:28,932 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 86/1265    Epoch: 0.14/4.0 (3.4% complete)    lr: 0.000925    \n",
      "2020-03-18 12:43:03,627 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 87/1265    Epoch: 0.14/4.0 (3.4% complete)    lr: 0.000924    \n",
      "2020-03-18 12:43:38,616 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 88/1265    Epoch: 0.14/4.0 (3.5% complete)    lr: 0.000923    \n",
      "2020-03-18 12:44:12,372 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 89/1265    Epoch: 0.14/4.0 (3.5% complete)    lr: 0.000922    \n",
      "2020-03-18 12:44:46,397 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 90/1265    Epoch: 0.14/4.0 (3.6% complete)    lr: 0.000921    \n",
      "2020-03-18 12:45:20,274 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 91/1265    Epoch: 0.14/4.0 (3.6% complete)    lr: 0.000921    \n",
      "2020-03-18 12:45:54,711 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 92/1265    Epoch: 0.15/4.0 (3.6% complete)    lr: 0.000920    \n",
      "2020-03-18 12:46:28,757 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 93/1265    Epoch: 0.15/4.0 (3.7% complete)    lr: 0.000919    \n",
      "2020-03-18 12:47:03,051 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 94/1265    Epoch: 0.15/4.0 (3.7% complete)    lr: 0.000918    \n",
      "2020-03-18 12:47:37,740 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 95/1265    Epoch: 0.15/4.0 (3.8% complete)    lr: 0.000917    \n",
      "2020-03-18 12:48:12,465 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 96/1265    Epoch: 0.15/4.0 (3.8% complete)    lr: 0.000916    \n",
      "2020-03-18 12:48:45,671 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 97/1265    Epoch: 0.15/4.0 (3.8% complete)    lr: 0.000916    \n",
      "2020-03-18 12:49:20,208 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 98/1265    Epoch: 0.15/4.0 (3.9% complete)    lr: 0.000915    \n",
      "2020-03-18 12:49:54,979 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 99/1265    Epoch: 0.16/4.0 (3.9% complete)    lr: 0.000914    \n",
      "2020-03-18 12:50:29,582 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 100/1265    Epoch: 0.16/4.0 (3.9% complete)    lr: 0.000913    \n",
      "2020-03-18 12:51:04,546 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 101/1265    Epoch: 0.16/4.0 (4.0% complete)    lr: 0.000912    \n",
      "2020-03-18 12:51:38,592 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 102/1265    Epoch: 0.16/4.0 (4.0% complete)    lr: 0.000911    \n",
      "2020-03-18 12:52:13,214 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 103/1265    Epoch: 0.16/4.0 (4.1% complete)    lr: 0.000911    \n",
      "2020-03-18 12:52:47,569 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 104/1265    Epoch: 0.16/4.0 (4.1% complete)    lr: 0.000910    \n",
      "2020-03-18 12:53:22,229 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 105/1265    Epoch: 0.17/4.0 (4.1% complete)    lr: 0.000909    \n",
      "2020-03-18 12:53:57,039 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 106/1265    Epoch: 0.17/4.0 (4.2% complete)    lr: 0.000908    \n",
      "2020-03-18 12:54:31,440 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 107/1265    Epoch: 0.17/4.0 (4.2% complete)    lr: 0.000907    \n",
      "2020-03-18 12:55:05,966 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 108/1265    Epoch: 0.17/4.0 (4.3% complete)    lr: 0.000906    \n",
      "2020-03-18 12:55:40,278 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 109/1265    Epoch: 0.17/4.0 (4.3% complete)    lr: 0.000906    \n",
      "2020-03-18 12:56:14,791 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 110/1265    Epoch: 0.17/4.0 (4.3% complete)    lr: 0.000905    \n",
      "2020-03-18 12:56:49,370 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 111/1265    Epoch: 0.18/4.0 (4.4% complete)    lr: 0.000904    \n",
      "2020-03-18 12:57:23,877 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 112/1265    Epoch: 0.18/4.0 (4.4% complete)    lr: 0.000903    \n",
      "2020-03-18 12:57:58,498 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 113/1265    Epoch: 0.18/4.0 (4.5% complete)    lr: 0.000902    \n",
      "2020-03-18 12:58:32,593 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 114/1265    Epoch: 0.18/4.0 (4.5% complete)    lr: 0.000902    \n",
      "2020-03-18 12:59:07,288 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 115/1265    Epoch: 0.18/4.0 (4.5% complete)    lr: 0.000901    \n",
      "2020-03-18 12:59:41,740 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 116/1265    Epoch: 0.18/4.0 (4.6% complete)    lr: 0.000900    \n",
      "2020-03-18 13:00:16,060 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 117/1265    Epoch: 0.18/4.0 (4.6% complete)    lr: 0.000899    \n",
      "2020-03-18 13:00:50,726 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 118/1265    Epoch: 0.19/4.0 (4.7% complete)    lr: 0.000898    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 13:01:25,615 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 119/1265    Epoch: 0.19/4.0 (4.7% complete)    lr: 0.000897    \n",
      "2020-03-18 13:02:00,530 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 120/1265    Epoch: 0.19/4.0 (4.7% complete)    lr: 0.000897    \n",
      "2020-03-18 13:02:36,300 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 121/1265    Epoch: 0.19/4.0 (4.8% complete)    lr: 0.000896    \n",
      "2020-03-18 13:03:10,929 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 122/1265    Epoch: 0.19/4.0 (4.8% complete)    lr: 0.000895    \n",
      "2020-03-18 13:03:45,535 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 123/1265    Epoch: 0.19/4.0 (4.9% complete)    lr: 0.000894    \n",
      "2020-03-18 13:04:20,119 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 124/1265    Epoch: 0.20/4.0 (4.9% complete)    lr: 0.000893    \n",
      "2020-03-18 13:04:54,819 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 125/1265    Epoch: 0.20/4.0 (4.9% complete)    lr: 0.000893    \n",
      "2020-03-18 13:05:29,723 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 126/1265    Epoch: 0.20/4.0 (5.0% complete)    lr: 0.000892    \n",
      "2020-03-18 13:06:04,024 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 127/1265    Epoch: 0.20/4.0 (5.0% complete)    lr: 0.000891    \n",
      "2020-03-18 13:06:37,563 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 128/1265    Epoch: 0.20/4.0 (5.1% complete)    lr: 0.000890    \n",
      "2020-03-18 13:07:12,371 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 129/1265    Epoch: 0.20/4.0 (5.1% complete)    lr: 0.000889    \n",
      "2020-03-18 13:07:46,701 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 130/1265    Epoch: 0.21/4.0 (5.1% complete)    lr: 0.000888    \n",
      "2020-03-18 13:08:20,611 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 131/1265    Epoch: 0.21/4.0 (5.2% complete)    lr: 0.000888    \n",
      "2020-03-18 13:08:55,239 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 132/1265    Epoch: 0.21/4.0 (5.2% complete)    lr: 0.000887    \n",
      "2020-03-18 13:09:29,520 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 133/1265    Epoch: 0.21/4.0 (5.3% complete)    lr: 0.000886    \n",
      "2020-03-18 13:10:03,608 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 134/1265    Epoch: 0.21/4.0 (5.3% complete)    lr: 0.000885    \n",
      "2020-03-18 13:10:37,614 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 135/1265    Epoch: 0.21/4.0 (5.3% complete)    lr: 0.000884    \n",
      "2020-03-18 13:11:11,661 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 136/1265    Epoch: 0.21/4.0 (5.4% complete)    lr: 0.000884    \n",
      "2020-03-18 13:11:45,411 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 137/1265    Epoch: 0.22/4.0 (5.4% complete)    lr: 0.000883    \n",
      "2020-03-18 13:12:20,406 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 138/1265    Epoch: 0.22/4.0 (5.5% complete)    lr: 0.000882    \n",
      "2020-03-18 13:12:54,127 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 139/1265    Epoch: 0.22/4.0 (5.5% complete)    lr: 0.000881    \n",
      "2020-03-18 13:13:28,498 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 140/1265    Epoch: 0.22/4.0 (5.5% complete)    lr: 0.000880    \n",
      "2020-03-18 13:14:04,023 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 141/1265    Epoch: 0.22/4.0 (5.6% complete)    lr: 0.000880    \n",
      "2020-03-18 13:14:38,222 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 142/1265    Epoch: 0.22/4.0 (5.6% complete)    lr: 0.000879    \n",
      "2020-03-18 13:15:12,406 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 143/1265    Epoch: 0.23/4.0 (5.6% complete)    lr: 0.000878    \n",
      "2020-03-18 13:15:46,715 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 144/1265    Epoch: 0.23/4.0 (5.7% complete)    lr: 0.000877    \n",
      "2020-03-18 13:16:20,593 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 145/1265    Epoch: 0.23/4.0 (5.7% complete)    lr: 0.000876    \n",
      "2020-03-18 13:16:55,087 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 146/1265    Epoch: 0.23/4.0 (5.8% complete)    lr: 0.000876    \n",
      "2020-03-18 13:17:29,786 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 147/1265    Epoch: 0.23/4.0 (5.8% complete)    lr: 0.000875    \n",
      "2020-03-18 13:18:03,871 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 148/1265    Epoch: 0.23/4.0 (5.8% complete)    lr: 0.000874    \n",
      "2020-03-18 13:18:38,266 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 149/1265    Epoch: 0.24/4.0 (5.9% complete)    lr: 0.000873    \n",
      "2020-03-18 13:19:12,780 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 150/1265    Epoch: 0.24/4.0 (5.9% complete)    lr: 0.000872    \n",
      "2020-03-18 13:19:47,607 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 151/1265    Epoch: 0.24/4.0 (6.0% complete)    lr: 0.000872    \n",
      "2020-03-18 13:20:21,753 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 152/1265    Epoch: 0.24/4.0 (6.0% complete)    lr: 0.000871    \n",
      "2020-03-18 13:20:55,299 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 153/1265    Epoch: 0.24/4.0 (6.0% complete)    lr: 0.000870    \n",
      "2020-03-18 13:21:30,273 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 154/1265    Epoch: 0.24/4.0 (6.1% complete)    lr: 0.000869    \n",
      "2020-03-18 13:22:04,929 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 155/1265    Epoch: 0.24/4.0 (6.1% complete)    lr: 0.000869    \n",
      "2020-03-18 13:22:39,811 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 156/1265    Epoch: 0.25/4.0 (6.2% complete)    lr: 0.000868    \n",
      "2020-03-18 13:23:14,520 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 157/1265    Epoch: 0.25/4.0 (6.2% complete)    lr: 0.000867    \n",
      "2020-03-18 13:23:49,324 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 158/1265    Epoch: 0.25/4.0 (6.2% complete)    lr: 0.000866    \n",
      "2020-03-18 13:24:23,525 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 159/1265    Epoch: 0.25/4.0 (6.3% complete)    lr: 0.000865    \n",
      "2020-03-18 13:24:57,629 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 160/1265    Epoch: 0.25/4.0 (6.3% complete)    lr: 0.000865    \n",
      "2020-03-18 13:25:33,304 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 161/1265    Epoch: 0.25/4.0 (6.4% complete)    lr: 0.000864    \n",
      "2020-03-18 13:26:07,289 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 162/1265    Epoch: 0.26/4.0 (6.4% complete)    lr: 0.000863    \n",
      "2020-03-18 13:26:41,845 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 163/1265    Epoch: 0.26/4.0 (6.4% complete)    lr: 0.000862    \n",
      "2020-03-18 13:27:15,926 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 164/1265    Epoch: 0.26/4.0 (6.5% complete)    lr: 0.000861    \n",
      "2020-03-18 13:27:50,379 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 165/1265    Epoch: 0.26/4.0 (6.5% complete)    lr: 0.000861    \n",
      "2020-03-18 13:28:23,319 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 166/1265    Epoch: 0.26/4.0 (6.6% complete)    lr: 0.000860    \n",
      "2020-03-18 13:28:57,919 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 167/1265    Epoch: 0.26/4.0 (6.6% complete)    lr: 0.000859    \n",
      "2020-03-18 13:29:32,198 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 168/1265    Epoch: 0.27/4.0 (6.6% complete)    lr: 0.000858    \n",
      "2020-03-18 13:30:06,924 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 169/1265    Epoch: 0.27/4.0 (6.7% complete)    lr: 0.000858    \n",
      "2020-03-18 13:30:40,774 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 170/1265    Epoch: 0.27/4.0 (6.7% complete)    lr: 0.000857    \n",
      "2020-03-18 13:31:15,631 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 171/1265    Epoch: 0.27/4.0 (6.8% complete)    lr: 0.000856    \n",
      "2020-03-18 13:31:50,189 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 172/1265    Epoch: 0.27/4.0 (6.8% complete)    lr: 0.000855    \n",
      "2020-03-18 13:32:24,139 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 173/1265    Epoch: 0.27/4.0 (6.8% complete)    lr: 0.000854    \n",
      "2020-03-18 13:32:59,265 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 174/1265    Epoch: 0.27/4.0 (6.9% complete)    lr: 0.000854    \n",
      "2020-03-18 13:33:32,834 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 175/1265    Epoch: 0.28/4.0 (6.9% complete)    lr: 0.000853    \n",
      "2020-03-18 13:34:07,586 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 176/1265    Epoch: 0.28/4.0 (7.0% complete)    lr: 0.000852    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 13:34:42,186 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 177/1265    Epoch: 0.28/4.0 (7.0% complete)    lr: 0.000851    \n",
      "2020-03-18 13:35:17,129 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 178/1265    Epoch: 0.28/4.0 (7.0% complete)    lr: 0.000851    \n",
      "2020-03-18 13:35:51,097 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 179/1265    Epoch: 0.28/4.0 (7.1% complete)    lr: 0.000850    \n",
      "2020-03-18 13:36:25,725 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 180/1265    Epoch: 0.28/4.0 (7.1% complete)    lr: 0.000849    \n",
      "2020-03-18 13:37:00,366 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 181/1265    Epoch: 0.29/4.0 (7.1% complete)    lr: 0.000848    \n",
      "2020-03-18 13:37:34,933 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 182/1265    Epoch: 0.29/4.0 (7.2% complete)    lr: 0.000847    \n",
      "2020-03-18 13:38:10,040 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 183/1265    Epoch: 0.29/4.0 (7.2% complete)    lr: 0.000847    \n",
      "2020-03-18 13:38:44,732 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 184/1265    Epoch: 0.29/4.0 (7.3% complete)    lr: 0.000846    \n",
      "2020-03-18 13:39:19,522 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 185/1265    Epoch: 0.29/4.0 (7.3% complete)    lr: 0.000845    \n",
      "2020-03-18 13:39:53,640 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 186/1265    Epoch: 0.29/4.0 (7.3% complete)    lr: 0.000844    \n",
      "2020-03-18 13:40:27,466 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 187/1265    Epoch: 0.30/4.0 (7.4% complete)    lr: 0.000844    \n",
      "2020-03-18 13:41:00,860 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 188/1265    Epoch: 0.30/4.0 (7.4% complete)    lr: 0.000843    \n",
      "2020-03-18 13:41:35,214 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 189/1265    Epoch: 0.30/4.0 (7.5% complete)    lr: 0.000842    \n",
      "2020-03-18 13:42:10,131 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 190/1265    Epoch: 0.30/4.0 (7.5% complete)    lr: 0.000841    \n",
      "2020-03-18 13:42:44,831 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 191/1265    Epoch: 0.30/4.0 (7.5% complete)    lr: 0.000841    \n",
      "2020-03-18 13:43:19,849 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 192/1265    Epoch: 0.30/4.0 (7.6% complete)    lr: 0.000840    \n",
      "2020-03-18 13:43:54,121 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 193/1265    Epoch: 0.30/4.0 (7.6% complete)    lr: 0.000839    \n",
      "2020-03-18 13:44:28,495 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 194/1265    Epoch: 0.31/4.0 (7.7% complete)    lr: 0.000838    \n",
      "2020-03-18 13:45:02,317 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 195/1265    Epoch: 0.31/4.0 (7.7% complete)    lr: 0.000838    \n",
      "2020-03-18 13:45:37,177 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 196/1265    Epoch: 0.31/4.0 (7.7% complete)    lr: 0.000837    \n",
      "2020-03-18 13:46:12,228 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 197/1265    Epoch: 0.31/4.0 (7.8% complete)    lr: 0.000836    \n",
      "2020-03-18 13:46:46,440 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 198/1265    Epoch: 0.31/4.0 (7.8% complete)    lr: 0.000835    \n",
      "2020-03-18 13:47:20,275 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 199/1265    Epoch: 0.31/4.0 (7.9% complete)    lr: 0.000834    \n",
      "2020-03-18 13:47:54,437 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 200/1265    Epoch: 0.32/4.0 (7.9% complete)    lr: 0.000834    \n",
      "2020-03-18 13:48:29,586 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 201/1265    Epoch: 0.32/4.0 (7.9% complete)    lr: 0.000833    \n",
      "2020-03-18 13:49:05,027 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 202/1265    Epoch: 0.32/4.0 (8.0% complete)    lr: 0.000832    \n",
      "2020-03-18 13:49:40,076 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 203/1265    Epoch: 0.32/4.0 (8.0% complete)    lr: 0.000831    \n",
      "2020-03-18 13:50:13,965 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 204/1265    Epoch: 0.32/4.0 (8.1% complete)    lr: 0.000831    \n",
      "2020-03-18 13:50:48,584 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 205/1265    Epoch: 0.32/4.0 (8.1% complete)    lr: 0.000830    \n",
      "2020-03-18 13:51:23,223 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 206/1265    Epoch: 0.33/4.0 (8.1% complete)    lr: 0.000829    \n",
      "2020-03-18 13:51:57,793 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 207/1265    Epoch: 0.33/4.0 (8.2% complete)    lr: 0.000828    \n",
      "2020-03-18 13:52:30,724 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 208/1265    Epoch: 0.33/4.0 (8.2% complete)    lr: 0.000828    \n",
      "2020-03-18 13:53:05,232 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 209/1265    Epoch: 0.33/4.0 (8.3% complete)    lr: 0.000827    \n",
      "2020-03-18 13:53:40,939 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 210/1265    Epoch: 0.33/4.0 (8.3% complete)    lr: 0.000826    \n",
      "2020-03-18 13:54:15,670 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 211/1265    Epoch: 0.33/4.0 (8.3% complete)    lr: 0.000825    \n",
      "2020-03-18 13:54:49,220 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 212/1265    Epoch: 0.33/4.0 (8.4% complete)    lr: 0.000825    \n",
      "2020-03-18 13:55:22,607 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 213/1265    Epoch: 0.34/4.0 (8.4% complete)    lr: 0.000824    \n",
      "2020-03-18 13:55:57,354 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 214/1265    Epoch: 0.34/4.0 (8.5% complete)    lr: 0.000823    \n",
      "2020-03-18 13:56:31,818 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 215/1265    Epoch: 0.34/4.0 (8.5% complete)    lr: 0.000822    \n",
      "2020-03-18 13:57:05,978 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 216/1265    Epoch: 0.34/4.0 (8.5% complete)    lr: 0.000822    \n",
      "2020-03-18 13:57:40,938 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 217/1265    Epoch: 0.34/4.0 (8.6% complete)    lr: 0.000821    \n",
      "2020-03-18 13:58:15,885 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 218/1265    Epoch: 0.34/4.0 (8.6% complete)    lr: 0.000820    \n",
      "2020-03-18 13:58:49,790 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 219/1265    Epoch: 0.35/4.0 (8.6% complete)    lr: 0.000819    \n",
      "2020-03-18 13:59:23,778 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 220/1265    Epoch: 0.35/4.0 (8.7% complete)    lr: 0.000819    \n",
      "2020-03-18 13:59:58,970 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 221/1265    Epoch: 0.35/4.0 (8.7% complete)    lr: 0.000818    \n",
      "2020-03-18 14:00:33,728 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 222/1265    Epoch: 0.35/4.0 (8.8% complete)    lr: 0.000817    \n",
      "2020-03-18 14:01:08,240 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 223/1265    Epoch: 0.35/4.0 (8.8% complete)    lr: 0.000816    \n",
      "2020-03-18 14:01:43,549 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 224/1265    Epoch: 0.35/4.0 (8.8% complete)    lr: 0.000816    \n",
      "2020-03-18 14:02:18,833 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 225/1265    Epoch: 0.36/4.0 (8.9% complete)    lr: 0.000815    \n",
      "2020-03-18 14:02:54,837 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 226/1265    Epoch: 0.36/4.0 (8.9% complete)    lr: 0.000814    \n",
      "2020-03-18 14:03:30,763 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 227/1265    Epoch: 0.36/4.0 (9.0% complete)    lr: 0.000813    \n",
      "2020-03-18 14:04:05,359 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 228/1265    Epoch: 0.36/4.0 (9.0% complete)    lr: 0.000813    \n",
      "2020-03-18 14:04:40,159 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 229/1265    Epoch: 0.36/4.0 (9.0% complete)    lr: 0.000812    \n",
      "2020-03-18 14:05:14,375 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 230/1265    Epoch: 0.36/4.0 (9.1% complete)    lr: 0.000811    \n",
      "2020-03-18 14:05:49,045 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 231/1265    Epoch: 0.36/4.0 (9.1% complete)    lr: 0.000811    \n",
      "2020-03-18 14:06:23,741 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 232/1265    Epoch: 0.37/4.0 (9.2% complete)    lr: 0.000810    \n",
      "2020-03-18 14:06:59,451 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 233/1265    Epoch: 0.37/4.0 (9.2% complete)    lr: 0.000809    \n",
      "2020-03-18 14:07:33,040 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 234/1265    Epoch: 0.37/4.0 (9.2% complete)    lr: 0.000808    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 14:08:07,698 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 235/1265    Epoch: 0.37/4.0 (9.3% complete)    lr: 0.000808    \n",
      "2020-03-18 14:08:42,182 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 236/1265    Epoch: 0.37/4.0 (9.3% complete)    lr: 0.000807    \n",
      "2020-03-18 14:09:16,964 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 237/1265    Epoch: 0.37/4.0 (9.4% complete)    lr: 0.000806    \n",
      "2020-03-18 14:09:51,634 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 238/1265    Epoch: 0.38/4.0 (9.4% complete)    lr: 0.000805    \n",
      "2020-03-18 14:10:26,970 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 239/1265    Epoch: 0.38/4.0 (9.4% complete)    lr: 0.000805    \n",
      "2020-03-18 14:11:01,068 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 240/1265    Epoch: 0.38/4.0 (9.5% complete)    lr: 0.000804    \n",
      "2020-03-18 14:11:37,905 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 241/1265    Epoch: 0.38/4.0 (9.5% complete)    lr: 0.000803    \n",
      "2020-03-18 14:12:13,562 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 242/1265    Epoch: 0.38/4.0 (9.6% complete)    lr: 0.000802    \n",
      "2020-03-18 14:12:48,029 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 243/1265    Epoch: 0.38/4.0 (9.6% complete)    lr: 0.000802    \n",
      "2020-03-18 14:13:22,785 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 244/1265    Epoch: 0.39/4.0 (9.6% complete)    lr: 0.000801    \n",
      "2020-03-18 14:13:56,535 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 245/1265    Epoch: 0.39/4.0 (9.7% complete)    lr: 0.000800    \n",
      "2020-03-18 14:14:31,086 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 246/1265    Epoch: 0.39/4.0 (9.7% complete)    lr: 0.000800    \n",
      "2020-03-18 14:15:05,109 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 247/1265    Epoch: 0.39/4.0 (9.8% complete)    lr: 0.000799    \n",
      "2020-03-18 14:15:38,828 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 248/1265    Epoch: 0.39/4.0 (9.8% complete)    lr: 0.000798    \n",
      "2020-03-18 14:16:13,454 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 249/1265    Epoch: 0.39/4.0 (9.8% complete)    lr: 0.000797    \n",
      "2020-03-18 14:16:47,754 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 250/1265    Epoch: 0.39/4.0 (9.9% complete)    lr: 0.000797    \n",
      "2020-03-18 14:17:21,464 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 251/1265    Epoch: 0.40/4.0 (9.9% complete)    lr: 0.000796    \n",
      "2020-03-18 14:17:56,678 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 252/1265    Epoch: 0.40/4.0 (10.0% complete)    lr: 0.000795    \n",
      "2020-03-18 14:18:29,876 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 253/1265    Epoch: 0.40/4.0 (10.0% complete)    lr: 0.000794    \n",
      "2020-03-18 14:19:04,470 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 254/1265    Epoch: 0.40/4.0 (10.0% complete)    lr: 0.000794    \n",
      "2020-03-18 14:19:39,495 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 255/1265    Epoch: 0.40/4.0 (10.1% complete)    lr: 0.000793    \n",
      "2020-03-18 14:20:12,107 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 256/1265    Epoch: 0.40/4.0 (10.1% complete)    lr: 0.000792    \n",
      "2020-03-18 14:20:46,828 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 257/1265    Epoch: 0.41/4.0 (10.2% complete)    lr: 0.000792    \n",
      "2020-03-18 14:21:20,708 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 258/1265    Epoch: 0.41/4.0 (10.2% complete)    lr: 0.000791    \n",
      "2020-03-18 14:21:55,655 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 259/1265    Epoch: 0.41/4.0 (10.2% complete)    lr: 0.000790    \n",
      "2020-03-18 14:22:30,097 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 260/1265    Epoch: 0.41/4.0 (10.3% complete)    lr: 0.000789    \n",
      "2020-03-18 14:23:05,165 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 261/1265    Epoch: 0.41/4.0 (10.3% complete)    lr: 0.000789    \n",
      "2020-03-18 14:23:42,118 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 262/1265    Epoch: 0.41/4.0 (10.3% complete)    lr: 0.000788    \n",
      "2020-03-18 14:24:19,663 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 263/1265    Epoch: 0.42/4.0 (10.4% complete)    lr: 0.000787    \n",
      "2020-03-18 14:24:55,429 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 264/1265    Epoch: 0.42/4.0 (10.4% complete)    lr: 0.000787    \n",
      "2020-03-18 14:25:31,051 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 265/1265    Epoch: 0.42/4.0 (10.5% complete)    lr: 0.000786    \n",
      "2020-03-18 14:26:05,438 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 266/1265    Epoch: 0.42/4.0 (10.5% complete)    lr: 0.000785    \n",
      "2020-03-18 14:26:39,575 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 267/1265    Epoch: 0.42/4.0 (10.5% complete)    lr: 0.000784    \n",
      "2020-03-18 14:27:14,937 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 268/1265    Epoch: 0.42/4.0 (10.6% complete)    lr: 0.000784    \n",
      "2020-03-18 14:27:49,469 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 269/1265    Epoch: 0.42/4.0 (10.6% complete)    lr: 0.000783    \n",
      "2020-03-18 14:28:25,675 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 270/1265    Epoch: 0.43/4.0 (10.7% complete)    lr: 0.000782    \n",
      "2020-03-18 14:28:59,949 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 271/1265    Epoch: 0.43/4.0 (10.7% complete)    lr: 0.000782    \n",
      "2020-03-18 14:29:34,312 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 272/1265    Epoch: 0.43/4.0 (10.7% complete)    lr: 0.000781    \n",
      "2020-03-18 14:30:08,397 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 273/1265    Epoch: 0.43/4.0 (10.8% complete)    lr: 0.000780    \n",
      "2020-03-18 14:30:43,564 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 274/1265    Epoch: 0.43/4.0 (10.8% complete)    lr: 0.000779    \n",
      "2020-03-18 14:31:15,321 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 275/1265    Epoch: 0.43/4.0 (10.9% complete)    lr: 0.000779    \n",
      "2020-03-18 14:31:49,568 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 276/1265    Epoch: 0.44/4.0 (10.9% complete)    lr: 0.000778    \n",
      "2020-03-18 14:32:24,387 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 277/1265    Epoch: 0.44/4.0 (10.9% complete)    lr: 0.000777    \n",
      "2020-03-18 14:32:58,299 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 278/1265    Epoch: 0.44/4.0 (11.0% complete)    lr: 0.000777    \n",
      "2020-03-18 14:33:33,133 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 279/1265    Epoch: 0.44/4.0 (11.0% complete)    lr: 0.000776    \n",
      "2020-03-18 14:34:08,346 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 280/1265    Epoch: 0.44/4.0 (11.1% complete)    lr: 0.000775    \n",
      "2020-03-18 14:34:48,401 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 281/1265    Epoch: 0.44/4.0 (11.1% complete)    lr: 0.000774    \n",
      "2020-03-18 14:35:20,992 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 282/1265    Epoch: 0.45/4.0 (11.1% complete)    lr: 0.000774    \n",
      "2020-03-18 14:35:54,120 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 283/1265    Epoch: 0.45/4.0 (11.2% complete)    lr: 0.000773    \n",
      "2020-03-18 14:36:29,297 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 284/1265    Epoch: 0.45/4.0 (11.2% complete)    lr: 0.000772    \n",
      "2020-03-18 14:37:04,452 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 285/1265    Epoch: 0.45/4.0 (11.3% complete)    lr: 0.000772    \n",
      "2020-03-18 14:37:39,287 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 286/1265    Epoch: 0.45/4.0 (11.3% complete)    lr: 0.000771    \n",
      "2020-03-18 14:38:12,419 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 287/1265    Epoch: 0.45/4.0 (11.3% complete)    lr: 0.000770    \n",
      "2020-03-18 14:38:47,835 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 288/1265    Epoch: 0.45/4.0 (11.4% complete)    lr: 0.000770    \n",
      "2020-03-18 14:39:22,780 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 289/1265    Epoch: 0.46/4.0 (11.4% complete)    lr: 0.000769    \n",
      "2020-03-18 14:39:58,414 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 290/1265    Epoch: 0.46/4.0 (11.5% complete)    lr: 0.000768    \n",
      "2020-03-18 14:40:31,976 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 291/1265    Epoch: 0.46/4.0 (11.5% complete)    lr: 0.000767    \n",
      "2020-03-18 14:41:04,204 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 292/1265    Epoch: 0.46/4.0 (11.5% complete)    lr: 0.000767    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 14:41:40,231 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 293/1265    Epoch: 0.46/4.0 (11.6% complete)    lr: 0.000766    \n",
      "2020-03-18 14:42:14,243 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 294/1265    Epoch: 0.46/4.0 (11.6% complete)    lr: 0.000765    \n",
      "2020-03-18 14:42:47,271 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 295/1265    Epoch: 0.47/4.0 (11.7% complete)    lr: 0.000765    \n",
      "2020-03-18 14:43:21,630 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 296/1265    Epoch: 0.47/4.0 (11.7% complete)    lr: 0.000764    \n",
      "2020-03-18 14:43:56,415 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 297/1265    Epoch: 0.47/4.0 (11.7% complete)    lr: 0.000763    \n",
      "2020-03-18 14:44:30,874 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 298/1265    Epoch: 0.47/4.0 (11.8% complete)    lr: 0.000763    \n",
      "2020-03-18 14:45:06,585 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 299/1265    Epoch: 0.47/4.0 (11.8% complete)    lr: 0.000762    \n",
      "2020-03-18 14:45:41,847 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 300/1265    Epoch: 0.47/4.0 (11.8% complete)    lr: 0.000761    \n",
      "2020-03-18 14:46:16,245 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 301/1265    Epoch: 0.48/4.0 (11.9% complete)    lr: 0.000761    \n",
      "2020-03-18 14:46:50,231 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 302/1265    Epoch: 0.48/4.0 (11.9% complete)    lr: 0.000760    \n",
      "2020-03-18 14:47:23,832 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 303/1265    Epoch: 0.48/4.0 (12.0% complete)    lr: 0.000759    \n",
      "2020-03-18 14:47:57,476 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 304/1265    Epoch: 0.48/4.0 (12.0% complete)    lr: 0.000758    \n",
      "2020-03-18 14:48:30,880 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 305/1265    Epoch: 0.48/4.0 (12.0% complete)    lr: 0.000758    \n",
      "2020-03-18 14:49:03,217 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 306/1265    Epoch: 0.48/4.0 (12.1% complete)    lr: 0.000757    \n",
      "2020-03-18 14:49:37,529 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 307/1265    Epoch: 0.48/4.0 (12.1% complete)    lr: 0.000756    \n",
      "2020-03-18 14:50:12,479 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 308/1265    Epoch: 0.49/4.0 (12.2% complete)    lr: 0.000756    \n",
      "2020-03-18 14:50:47,370 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 309/1265    Epoch: 0.49/4.0 (12.2% complete)    lr: 0.000755    \n",
      "2020-03-18 14:51:22,345 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 310/1265    Epoch: 0.49/4.0 (12.2% complete)    lr: 0.000754    \n",
      "2020-03-18 14:51:57,039 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 311/1265    Epoch: 0.49/4.0 (12.3% complete)    lr: 0.000754    \n",
      "2020-03-18 14:52:31,184 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 312/1265    Epoch: 0.49/4.0 (12.3% complete)    lr: 0.000753    \n",
      "2020-03-18 14:53:06,231 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 313/1265    Epoch: 0.49/4.0 (12.4% complete)    lr: 0.000752    \n",
      "2020-03-18 14:53:40,988 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 314/1265    Epoch: 0.50/4.0 (12.4% complete)    lr: 0.000752    \n",
      "2020-03-18 14:54:16,457 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 315/1265    Epoch: 0.50/4.0 (12.4% complete)    lr: 0.000751    \n",
      "2020-03-18 14:54:51,665 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 316/1265    Epoch: 0.50/4.0 (12.5% complete)    lr: 0.000750    \n",
      "2020-03-18 14:55:27,414 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 317/1265    Epoch: 0.50/4.0 (12.5% complete)    lr: 0.001499    \n",
      "2020-03-18 14:56:00,558 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 318/1265    Epoch: 0.50/4.0 (12.6% complete)    lr: 0.001496    \n",
      "2020-03-18 14:56:35,425 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 319/1265    Epoch: 0.51/4.0 (12.7% complete)    lr: 0.001494    \n",
      "2020-03-18 14:57:09,127 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 320/1265    Epoch: 0.51/4.0 (12.8% complete)    lr: 0.001491    \n",
      "2020-03-18 14:57:45,276 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 321/1265    Epoch: 0.51/4.0 (12.8% complete)    lr: 0.001488    \n",
      "2020-03-18 14:58:20,277 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 322/1265    Epoch: 0.52/4.0 (12.9% complete)    lr: 0.001486    \n",
      "2020-03-18 14:58:56,082 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 323/1265    Epoch: 0.52/4.0 (13.0% complete)    lr: 0.001483    \n",
      "2020-03-18 14:59:29,357 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 324/1265    Epoch: 0.52/4.0 (13.1% complete)    lr: 0.001480    \n",
      "2020-03-18 15:00:05,171 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 325/1265    Epoch: 0.53/4.0 (13.2% complete)    lr: 0.001477    \n",
      "2020-03-18 15:00:37,869 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 326/1265    Epoch: 0.53/4.0 (13.2% complete)    lr: 0.001475    \n",
      "2020-03-18 15:01:12,076 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 327/1265    Epoch: 0.53/4.0 (13.3% complete)    lr: 0.001472    \n",
      "2020-03-18 15:01:45,943 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 328/1265    Epoch: 0.54/4.0 (13.4% complete)    lr: 0.001469    \n",
      "2020-03-18 15:02:20,047 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 329/1265    Epoch: 0.54/4.0 (13.5% complete)    lr: 0.001467    \n",
      "2020-03-18 15:02:54,147 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 330/1265    Epoch: 0.54/4.0 (13.5% complete)    lr: 0.001464    \n",
      "2020-03-18 15:03:30,270 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 331/1265    Epoch: 0.55/4.0 (13.6% complete)    lr: 0.001461    \n",
      "2020-03-18 15:04:09,315 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 332/1265    Epoch: 0.55/4.0 (13.7% complete)    lr: 0.001459    \n",
      "2020-03-18 15:04:42,115 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 333/1265    Epoch: 0.55/4.0 (13.8% complete)    lr: 0.001456    \n",
      "2020-03-18 15:05:16,127 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 334/1265    Epoch: 0.55/4.0 (13.9% complete)    lr: 0.001453    \n",
      "2020-03-18 15:05:49,044 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 335/1265    Epoch: 0.56/4.0 (13.9% complete)    lr: 0.001451    \n",
      "2020-03-18 15:06:25,082 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 336/1265    Epoch: 0.56/4.0 (14.0% complete)    lr: 0.001448    \n",
      "2020-03-18 15:06:58,130 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 337/1265    Epoch: 0.56/4.0 (14.1% complete)    lr: 0.001446    \n",
      "2020-03-18 15:07:33,761 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 338/1265    Epoch: 0.57/4.0 (14.2% complete)    lr: 0.001443    \n",
      "2020-03-18 15:08:09,628 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 339/1265    Epoch: 0.57/4.0 (14.3% complete)    lr: 0.001440    \n",
      "2020-03-18 15:08:44,042 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 340/1265    Epoch: 0.57/4.0 (14.3% complete)    lr: 0.001438    \n",
      "2020-03-18 15:09:23,619 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 341/1265    Epoch: 0.58/4.0 (14.4% complete)    lr: 0.001435    \n",
      "2020-03-18 15:09:59,190 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 342/1265    Epoch: 0.58/4.0 (14.5% complete)    lr: 0.001432    \n",
      "2020-03-18 15:10:34,765 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 343/1265    Epoch: 0.58/4.0 (14.6% complete)    lr: 0.001430    \n",
      "2020-03-18 15:11:10,947 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 344/1265    Epoch: 0.59/4.0 (14.7% complete)    lr: 0.001427    \n",
      "2020-03-18 15:11:45,637 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 345/1265    Epoch: 0.59/4.0 (14.7% complete)    lr: 0.001425    \n",
      "2020-03-18 15:12:21,724 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 346/1265    Epoch: 0.59/4.0 (14.8% complete)    lr: 0.001422    \n",
      "2020-03-18 15:12:54,076 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 347/1265    Epoch: 0.60/4.0 (14.9% complete)    lr: 0.001420    \n",
      "2020-03-18 15:13:28,518 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 348/1265    Epoch: 0.60/4.0 (15.0% complete)    lr: 0.001417    \n",
      "2020-03-18 15:14:04,421 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 349/1265    Epoch: 0.60/4.0 (15.0% complete)    lr: 0.001414    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 15:14:38,283 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 350/1265    Epoch: 0.61/4.0 (15.1% complete)    lr: 0.001412    \n",
      "2020-03-18 15:15:13,147 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 351/1265    Epoch: 0.61/4.0 (15.2% complete)    lr: 0.001409    \n",
      "2020-03-18 15:15:50,481 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 352/1265    Epoch: 0.61/4.0 (15.3% complete)    lr: 0.001407    \n",
      "2020-03-18 15:16:23,166 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 353/1265    Epoch: 0.61/4.0 (15.4% complete)    lr: 0.001404    \n",
      "2020-03-18 15:16:58,805 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 354/1265    Epoch: 0.62/4.0 (15.4% complete)    lr: 0.001402    \n",
      "2020-03-18 15:17:32,744 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 355/1265    Epoch: 0.62/4.0 (15.5% complete)    lr: 0.001399    \n",
      "2020-03-18 15:18:06,081 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 356/1265    Epoch: 0.62/4.0 (15.6% complete)    lr: 0.001396    \n",
      "2020-03-18 15:18:41,935 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 357/1265    Epoch: 0.63/4.0 (15.7% complete)    lr: 0.001394    \n",
      "2020-03-18 15:19:17,794 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 358/1265    Epoch: 0.63/4.0 (15.8% complete)    lr: 0.001391    \n",
      "2020-03-18 15:19:51,756 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 359/1265    Epoch: 0.63/4.0 (15.8% complete)    lr: 0.001389    \n",
      "2020-03-18 15:20:28,259 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 360/1265    Epoch: 0.64/4.0 (15.9% complete)    lr: 0.001386    \n",
      "2020-03-18 15:21:04,643 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 361/1265    Epoch: 0.64/4.0 (16.0% complete)    lr: 0.001384    \n",
      "2020-03-18 15:21:40,520 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 362/1265    Epoch: 0.64/4.0 (16.1% complete)    lr: 0.001381    \n",
      "2020-03-18 15:22:15,765 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 363/1265    Epoch: 0.65/4.0 (16.2% complete)    lr: 0.001379    \n",
      "2020-03-18 15:22:52,182 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 364/1265    Epoch: 0.65/4.0 (16.2% complete)    lr: 0.001376    \n",
      "2020-03-18 15:23:28,543 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 365/1265    Epoch: 0.65/4.0 (16.3% complete)    lr: 0.001374    \n",
      "2020-03-18 15:24:06,309 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 366/1265    Epoch: 0.66/4.0 (16.4% complete)    lr: 0.001371    \n",
      "2020-03-18 15:24:43,791 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 367/1265    Epoch: 0.66/4.0 (16.5% complete)    lr: 0.001369    \n",
      "2020-03-18 15:25:18,968 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 368/1265    Epoch: 0.66/4.0 (16.5% complete)    lr: 0.001366    \n",
      "2020-03-18 15:25:56,001 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 369/1265    Epoch: 0.67/4.0 (16.6% complete)    lr: 0.001364    \n",
      "2020-03-18 15:26:32,180 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 370/1265    Epoch: 0.67/4.0 (16.7% complete)    lr: 0.001361    \n",
      "2020-03-18 15:27:05,276 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 371/1265    Epoch: 0.67/4.0 (16.8% complete)    lr: 0.001359    \n",
      "2020-03-18 15:27:41,719 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 372/1265    Epoch: 0.67/4.0 (16.9% complete)    lr: 0.001356    \n",
      "2020-03-18 15:28:15,174 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 373/1265    Epoch: 0.68/4.0 (16.9% complete)    lr: 0.001354    \n",
      "2020-03-18 15:28:50,350 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 374/1265    Epoch: 0.68/4.0 (17.0% complete)    lr: 0.001351    \n",
      "2020-03-18 15:29:24,921 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 375/1265    Epoch: 0.68/4.0 (17.1% complete)    lr: 0.001349    \n",
      "2020-03-18 15:30:00,458 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 376/1265    Epoch: 0.69/4.0 (17.2% complete)    lr: 0.001347    \n",
      "2020-03-18 15:30:36,040 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 377/1265    Epoch: 0.69/4.0 (17.3% complete)    lr: 0.001344    \n",
      "2020-03-18 15:31:09,643 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 378/1265    Epoch: 0.69/4.0 (17.3% complete)    lr: 0.001342    \n",
      "2020-03-18 15:31:44,268 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 379/1265    Epoch: 0.70/4.0 (17.4% complete)    lr: 0.001339    \n",
      "2020-03-18 15:32:20,109 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 380/1265    Epoch: 0.70/4.0 (17.5% complete)    lr: 0.001337    \n",
      "2020-03-18 15:32:58,916 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 381/1265    Epoch: 0.70/4.0 (17.6% complete)    lr: 0.001334    \n",
      "2020-03-18 15:33:33,543 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 382/1265    Epoch: 0.71/4.0 (17.7% complete)    lr: 0.001332    \n",
      "2020-03-18 15:34:07,120 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 383/1265    Epoch: 0.71/4.0 (17.7% complete)    lr: 0.001330    \n",
      "2020-03-18 15:34:42,407 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 384/1265    Epoch: 0.71/4.0 (17.8% complete)    lr: 0.001327    \n",
      "2020-03-18 15:35:17,759 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 385/1265    Epoch: 0.72/4.0 (17.9% complete)    lr: 0.001325    \n",
      "2020-03-18 15:35:52,036 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 386/1265    Epoch: 0.72/4.0 (18.0% complete)    lr: 0.001322    \n",
      "2020-03-18 15:36:29,496 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 387/1265    Epoch: 0.72/4.0 (18.0% complete)    lr: 0.001320    \n",
      "2020-03-18 15:37:03,974 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 388/1265    Epoch: 0.73/4.0 (18.1% complete)    lr: 0.001317    \n",
      "2020-03-18 15:37:37,990 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 389/1265    Epoch: 0.73/4.0 (18.2% complete)    lr: 0.001315    \n",
      "2020-03-18 15:38:14,167 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 390/1265    Epoch: 0.73/4.0 (18.3% complete)    lr: 0.001313    \n",
      "2020-03-18 15:38:50,991 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 391/1265    Epoch: 0.73/4.0 (18.4% complete)    lr: 0.001310    \n",
      "2020-03-18 15:39:27,106 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 392/1265    Epoch: 0.74/4.0 (18.4% complete)    lr: 0.001308    \n",
      "2020-03-18 15:40:01,140 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 393/1265    Epoch: 0.74/4.0 (18.5% complete)    lr: 0.001306    \n",
      "2020-03-18 15:40:35,659 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 394/1265    Epoch: 0.74/4.0 (18.6% complete)    lr: 0.001303    \n",
      "2020-03-18 15:41:10,422 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 395/1265    Epoch: 0.75/4.0 (18.7% complete)    lr: 0.001301    \n",
      "2020-03-18 15:41:44,656 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 396/1265    Epoch: 0.75/4.0 (18.8% complete)    lr: 0.001298    \n",
      "2020-03-18 15:42:20,114 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 397/1265    Epoch: 0.75/4.0 (18.8% complete)    lr: 0.001296    \n",
      "2020-03-18 15:42:55,252 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 398/1265    Epoch: 0.76/4.0 (18.9% complete)    lr: 0.001294    \n",
      "2020-03-18 15:43:29,820 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 399/1265    Epoch: 0.76/4.0 (19.0% complete)    lr: 0.001291    \n",
      "2020-03-18 15:44:04,974 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 400/1265    Epoch: 0.76/4.0 (19.1% complete)    lr: 0.001289    \n",
      "2020-03-18 15:44:41,951 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 401/1265    Epoch: 0.77/4.0 (19.2% complete)    lr: 0.001287    \n",
      "2020-03-18 15:45:15,100 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 402/1265    Epoch: 0.77/4.0 (19.2% complete)    lr: 0.001284    \n",
      "2020-03-18 15:45:50,119 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 403/1265    Epoch: 0.77/4.0 (19.3% complete)    lr: 0.001282    \n",
      "2020-03-18 15:46:24,663 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 404/1265    Epoch: 0.78/4.0 (19.4% complete)    lr: 0.001280    \n",
      "2020-03-18 15:47:04,560 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 405/1265    Epoch: 0.78/4.0 (19.5% complete)    lr: 0.001277    \n",
      "2020-03-18 15:47:39,849 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 406/1265    Epoch: 0.78/4.0 (19.5% complete)    lr: 0.001275    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 15:48:16,010 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 407/1265    Epoch: 0.79/4.0 (19.6% complete)    lr: 0.001273    \n",
      "2020-03-18 15:48:50,822 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 408/1265    Epoch: 0.79/4.0 (19.7% complete)    lr: 0.001270    \n",
      "2020-03-18 15:49:26,266 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 409/1265    Epoch: 0.79/4.0 (19.8% complete)    lr: 0.001268    \n",
      "2020-03-18 15:50:00,191 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 410/1265    Epoch: 0.79/4.0 (19.9% complete)    lr: 0.001266    \n",
      "2020-03-18 15:50:32,773 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 411/1265    Epoch: 0.80/4.0 (19.9% complete)    lr: 0.001264    \n",
      "2020-03-18 15:51:09,397 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 412/1265    Epoch: 0.80/4.0 (20.0% complete)    lr: 0.001261    \n",
      "2020-03-18 15:51:47,293 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 413/1265    Epoch: 0.80/4.0 (20.1% complete)    lr: 0.001259    \n",
      "2020-03-18 15:52:22,950 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 414/1265    Epoch: 0.81/4.0 (20.2% complete)    lr: 0.001257    \n",
      "2020-03-18 15:52:59,195 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 415/1265    Epoch: 0.81/4.0 (20.3% complete)    lr: 0.001254    \n",
      "2020-03-18 15:53:35,951 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 416/1265    Epoch: 0.81/4.0 (20.3% complete)    lr: 0.001252    \n",
      "2020-03-18 15:54:13,623 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 417/1265    Epoch: 0.82/4.0 (20.4% complete)    lr: 0.001250    \n",
      "2020-03-18 15:54:48,520 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 418/1265    Epoch: 0.82/4.0 (20.5% complete)    lr: 0.001248    \n",
      "2020-03-18 15:55:24,428 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 419/1265    Epoch: 0.82/4.0 (20.6% complete)    lr: 0.001245    \n",
      "2020-03-18 15:55:58,745 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 420/1265    Epoch: 0.83/4.0 (20.7% complete)    lr: 0.001243    \n",
      "2020-03-18 15:56:35,501 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 421/1265    Epoch: 0.83/4.0 (20.7% complete)    lr: 0.001241    \n",
      "2020-03-18 15:57:10,838 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 422/1265    Epoch: 0.83/4.0 (20.8% complete)    lr: 0.001238    \n",
      "2020-03-18 15:57:45,208 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 423/1265    Epoch: 0.84/4.0 (20.9% complete)    lr: 0.001236    \n",
      "2020-03-18 15:58:20,127 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 424/1265    Epoch: 0.84/4.0 (21.0% complete)    lr: 0.001234    \n",
      "2020-03-18 15:58:53,913 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 425/1265    Epoch: 0.84/4.0 (21.1% complete)    lr: 0.001232    \n",
      "2020-03-18 15:59:26,863 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 426/1265    Epoch: 0.85/4.0 (21.1% complete)    lr: 0.001230    \n",
      "2020-03-18 16:00:01,898 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 427/1265    Epoch: 0.85/4.0 (21.2% complete)    lr: 0.001227    \n",
      "2020-03-18 16:00:44,858 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 428/1265    Epoch: 0.85/4.0 (21.3% complete)    lr: 0.001225    \n",
      "2020-03-18 16:01:18,135 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 429/1265    Epoch: 0.85/4.0 (21.4% complete)    lr: 0.001223    \n",
      "2020-03-18 16:01:50,624 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 430/1265    Epoch: 0.86/4.0 (21.4% complete)    lr: 0.001221    \n",
      "2020-03-18 16:02:24,580 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 431/1265    Epoch: 0.86/4.0 (21.5% complete)    lr: 0.001218    \n",
      "2020-03-18 16:02:57,390 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 432/1265    Epoch: 0.86/4.0 (21.6% complete)    lr: 0.001216    \n",
      "2020-03-18 16:03:31,084 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 433/1265    Epoch: 0.87/4.0 (21.7% complete)    lr: 0.001214    \n",
      "2020-03-18 16:04:05,182 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 434/1265    Epoch: 0.87/4.0 (21.8% complete)    lr: 0.001212    \n",
      "2020-03-18 16:04:38,607 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 435/1265    Epoch: 0.87/4.0 (21.8% complete)    lr: 0.001210    \n",
      "2020-03-18 16:05:11,429 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 436/1265    Epoch: 0.88/4.0 (21.9% complete)    lr: 0.001207    \n",
      "2020-03-18 16:05:45,158 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 437/1265    Epoch: 0.88/4.0 (22.0% complete)    lr: 0.001205    \n",
      "2020-03-18 16:06:19,646 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 438/1265    Epoch: 0.88/4.0 (22.1% complete)    lr: 0.001203    \n",
      "2020-03-18 16:06:52,346 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 439/1265    Epoch: 0.89/4.0 (22.2% complete)    lr: 0.001201    \n",
      "2020-03-18 16:07:26,027 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 440/1265    Epoch: 0.89/4.0 (22.2% complete)    lr: 0.001199    \n",
      "2020-03-18 16:08:02,513 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 441/1265    Epoch: 0.89/4.0 (22.3% complete)    lr: 0.001196    \n",
      "2020-03-18 16:08:36,051 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 442/1265    Epoch: 0.90/4.0 (22.4% complete)    lr: 0.001194    \n",
      "2020-03-18 16:09:10,056 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 443/1265    Epoch: 0.90/4.0 (22.5% complete)    lr: 0.001192    \n",
      "2020-03-18 16:09:43,897 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 444/1265    Epoch: 0.90/4.0 (22.6% complete)    lr: 0.001190    \n",
      "2020-03-18 16:10:18,191 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 445/1265    Epoch: 0.91/4.0 (22.6% complete)    lr: 0.001188    \n",
      "2020-03-18 16:10:52,410 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 446/1265    Epoch: 0.91/4.0 (22.7% complete)    lr: 0.001186    \n",
      "2020-03-18 16:11:24,990 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 447/1265    Epoch: 0.91/4.0 (22.8% complete)    lr: 0.001183    \n",
      "2020-03-18 16:11:59,015 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 448/1265    Epoch: 0.91/4.0 (22.9% complete)    lr: 0.001181    \n",
      "2020-03-18 16:12:31,969 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 449/1265    Epoch: 0.92/4.0 (22.9% complete)    lr: 0.001179    \n",
      "2020-03-18 16:13:04,377 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 450/1265    Epoch: 0.92/4.0 (23.0% complete)    lr: 0.001177    \n",
      "2020-03-18 16:13:38,685 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 451/1265    Epoch: 0.92/4.0 (23.1% complete)    lr: 0.001175    \n",
      "2020-03-18 16:14:13,684 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 452/1265    Epoch: 0.93/4.0 (23.2% complete)    lr: 0.001173    \n",
      "2020-03-18 16:14:48,552 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 453/1265    Epoch: 0.93/4.0 (23.3% complete)    lr: 0.001171    \n",
      "2020-03-18 16:15:24,175 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 454/1265    Epoch: 0.93/4.0 (23.3% complete)    lr: 0.001168    \n",
      "2020-03-18 16:15:59,355 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 455/1265    Epoch: 0.94/4.0 (23.4% complete)    lr: 0.001166    \n",
      "2020-03-18 16:16:33,640 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 456/1265    Epoch: 0.94/4.0 (23.5% complete)    lr: 0.001164    \n",
      "2020-03-18 16:17:07,951 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 457/1265    Epoch: 0.94/4.0 (23.6% complete)    lr: 0.001162    \n",
      "2020-03-18 16:17:42,714 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 458/1265    Epoch: 0.95/4.0 (23.7% complete)    lr: 0.001160    \n",
      "2020-03-18 16:18:16,812 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 459/1265    Epoch: 0.95/4.0 (23.7% complete)    lr: 0.001158    \n",
      "2020-03-18 16:18:51,151 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 460/1265    Epoch: 0.95/4.0 (23.8% complete)    lr: 0.001156    \n",
      "2020-03-18 16:19:27,560 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 461/1265    Epoch: 0.96/4.0 (23.9% complete)    lr: 0.001154    \n",
      "2020-03-18 16:20:07,069 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 462/1265    Epoch: 0.96/4.0 (24.0% complete)    lr: 0.001152    \n",
      "2020-03-18 16:20:45,848 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 463/1265    Epoch: 0.96/4.0 (24.1% complete)    lr: 0.001149    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 16:21:27,377 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 464/1265    Epoch: 0.97/4.0 (24.1% complete)    lr: 0.001147    \n",
      "2020-03-18 16:22:10,950 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 465/1265    Epoch: 0.97/4.0 (24.2% complete)    lr: 0.001145    \n",
      "2020-03-18 16:22:58,064 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 466/1265    Epoch: 0.97/4.0 (24.3% complete)    lr: 0.001143    \n",
      "2020-03-18 16:23:39,045 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 467/1265    Epoch: 0.97/4.0 (24.4% complete)    lr: 0.001141    \n",
      "2020-03-18 16:24:20,151 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 468/1265    Epoch: 0.98/4.0 (24.4% complete)    lr: 0.001139    \n",
      "2020-03-18 16:24:55,706 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 469/1265    Epoch: 0.98/4.0 (24.5% complete)    lr: 0.001137    \n",
      "2020-03-18 16:25:34,335 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 470/1265    Epoch: 0.98/4.0 (24.6% complete)    lr: 0.001135    \n",
      "2020-03-18 16:26:10,541 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 471/1265    Epoch: 0.99/4.0 (24.7% complete)    lr: 0.001133    \n",
      "2020-03-18 16:26:46,934 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 472/1265    Epoch: 0.99/4.0 (24.8% complete)    lr: 0.001131    \n",
      "2020-03-18 16:27:21,942 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 473/1265    Epoch: 0.99/4.0 (24.8% complete)    lr: 0.001129    \n",
      "2020-03-18 16:27:57,721 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 474/1265    Epoch: 1.00/4.0 (24.9% complete)    lr: 0.001127    \n",
      "2020-03-18 16:28:32,758 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 475/1265    Epoch: 1.00/4.0 (25.0% complete)    lr: 0.001125    \n",
      "2020-03-18 16:29:07,512 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 476/1265    Epoch: 1.00/4.0 (25.1% complete)    lr: 0.001123    \n",
      "2020-03-18 16:29:48,421 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 477/1265    Epoch: 1.01/4.0 (25.2% complete)    lr: 0.001121    \n",
      "2020-03-18 16:30:34,408 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 478/1265    Epoch: 1.01/4.0 (25.2% complete)    lr: 0.001119    \n",
      "2020-03-18 16:31:18,235 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 479/1265    Epoch: 1.01/4.0 (25.3% complete)    lr: 0.001117    \n",
      "2020-03-18 16:31:57,935 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 480/1265    Epoch: 1.02/4.0 (25.4% complete)    lr: 0.001115    \n",
      "2020-03-18 16:32:44,857 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 481/1265    Epoch: 1.02/4.0 (25.5% complete)    lr: 0.001112    \n",
      "2020-03-18 16:33:36,646 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 482/1265    Epoch: 1.02/4.0 (25.6% complete)    lr: 0.001110    \n",
      "2020-03-18 16:34:24,997 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 483/1265    Epoch: 1.03/4.0 (25.6% complete)    lr: 0.001108    \n",
      "2020-03-18 16:35:07,276 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 484/1265    Epoch: 1.03/4.0 (25.7% complete)    lr: 0.001106    \n",
      "2020-03-18 16:35:47,196 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 485/1265    Epoch: 1.03/4.0 (25.8% complete)    lr: 0.001104    \n",
      "2020-03-18 16:36:31,552 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 486/1265    Epoch: 1.03/4.0 (25.9% complete)    lr: 0.001102    \n",
      "2020-03-18 16:37:09,572 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 487/1265    Epoch: 1.04/4.0 (25.9% complete)    lr: 0.001100    \n",
      "2020-03-18 16:37:59,831 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 488/1265    Epoch: 1.04/4.0 (26.0% complete)    lr: 0.001098    \n",
      "2020-03-18 16:38:50,697 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 489/1265    Epoch: 1.04/4.0 (26.1% complete)    lr: 0.001096    \n",
      "2020-03-18 16:39:38,181 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 490/1265    Epoch: 1.05/4.0 (26.2% complete)    lr: 0.001094    \n",
      "2020-03-18 16:40:22,524 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 491/1265    Epoch: 1.05/4.0 (26.3% complete)    lr: 0.001092    \n",
      "2020-03-18 16:41:11,944 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 492/1265    Epoch: 1.05/4.0 (26.3% complete)    lr: 0.001090    \n",
      "2020-03-18 16:42:04,394 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 493/1265    Epoch: 1.06/4.0 (26.4% complete)    lr: 0.001088    \n",
      "2020-03-18 16:42:54,019 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 494/1265    Epoch: 1.06/4.0 (26.5% complete)    lr: 0.001086    \n",
      "2020-03-18 16:43:39,044 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 495/1265    Epoch: 1.06/4.0 (26.6% complete)    lr: 0.001085    \n",
      "2020-03-18 16:44:21,257 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 496/1265    Epoch: 1.07/4.0 (26.7% complete)    lr: 0.001083    \n",
      "2020-03-18 16:45:13,045 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 497/1265    Epoch: 1.07/4.0 (26.7% complete)    lr: 0.001081    \n",
      "2020-03-18 16:45:51,821 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 498/1265    Epoch: 1.07/4.0 (26.8% complete)    lr: 0.001079    \n",
      "2020-03-18 16:46:30,634 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 499/1265    Epoch: 1.08/4.0 (26.9% complete)    lr: 0.001077    \n",
      "2020-03-18 16:47:16,771 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 500/1265    Epoch: 1.08/4.0 (27.0% complete)    lr: 0.001075    \n",
      "2020-03-18 16:47:53,343 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 501/1265    Epoch: 1.08/4.0 (27.1% complete)    lr: 0.001073    \n",
      "2020-03-18 16:48:31,796 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 502/1265    Epoch: 1.09/4.0 (27.1% complete)    lr: 0.001071    \n",
      "2020-03-18 16:49:10,153 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 503/1265    Epoch: 1.09/4.0 (27.2% complete)    lr: 0.001069    \n",
      "2020-03-18 16:49:51,384 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 504/1265    Epoch: 1.09/4.0 (27.3% complete)    lr: 0.001067    \n",
      "2020-03-18 16:50:29,475 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 505/1265    Epoch: 1.09/4.0 (27.4% complete)    lr: 0.001065    \n",
      "2020-03-18 16:51:03,880 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 506/1265    Epoch: 1.10/4.0 (27.4% complete)    lr: 0.001063    \n",
      "2020-03-18 16:51:38,982 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 507/1265    Epoch: 1.10/4.0 (27.5% complete)    lr: 0.001061    \n",
      "2020-03-18 16:52:12,334 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 508/1265    Epoch: 1.10/4.0 (27.6% complete)    lr: 0.001059    \n",
      "2020-03-18 16:52:45,275 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 509/1265    Epoch: 1.11/4.0 (27.7% complete)    lr: 0.001057    \n",
      "2020-03-18 16:53:17,759 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 510/1265    Epoch: 1.11/4.0 (27.8% complete)    lr: 0.001055    \n",
      "2020-03-18 16:53:53,507 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 511/1265    Epoch: 1.11/4.0 (27.8% complete)    lr: 0.001053    \n",
      "2020-03-18 16:54:28,179 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 512/1265    Epoch: 1.12/4.0 (27.9% complete)    lr: 0.001051    \n",
      "2020-03-18 16:55:02,519 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 513/1265    Epoch: 1.12/4.0 (28.0% complete)    lr: 0.001050    \n",
      "2020-03-18 16:55:34,989 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 514/1265    Epoch: 1.12/4.0 (28.1% complete)    lr: 0.001048    \n",
      "2020-03-18 16:56:10,202 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 515/1265    Epoch: 1.13/4.0 (28.2% complete)    lr: 0.001046    \n",
      "2020-03-18 16:56:42,259 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 516/1265    Epoch: 1.13/4.0 (28.2% complete)    lr: 0.001044    \n",
      "2020-03-18 16:57:16,245 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 517/1265    Epoch: 1.13/4.0 (28.3% complete)    lr: 0.001042    \n",
      "2020-03-18 16:57:50,860 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 518/1265    Epoch: 1.14/4.0 (28.4% complete)    lr: 0.001040    \n",
      "2020-03-18 16:58:24,769 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 519/1265    Epoch: 1.14/4.0 (28.5% complete)    lr: 0.001038    \n",
      "2020-03-18 16:58:59,075 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 520/1265    Epoch: 1.14/4.0 (28.6% complete)    lr: 0.001036    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 16:59:33,916 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 521/1265    Epoch: 1.15/4.0 (28.6% complete)    lr: 0.001034    \n",
      "2020-03-18 17:00:06,012 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 522/1265    Epoch: 1.15/4.0 (28.7% complete)    lr: 0.001033    \n",
      "2020-03-18 17:00:39,270 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 523/1265    Epoch: 1.15/4.0 (28.8% complete)    lr: 0.001031    \n",
      "2020-03-18 17:01:13,474 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 524/1265    Epoch: 1.15/4.0 (28.9% complete)    lr: 0.001029    \n",
      "2020-03-18 17:01:45,293 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 525/1265    Epoch: 1.16/4.0 (28.9% complete)    lr: 0.001027    \n",
      "2020-03-18 17:02:18,278 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 526/1265    Epoch: 1.16/4.0 (29.0% complete)    lr: 0.001025    \n",
      "2020-03-18 17:02:48,873 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 527/1265    Epoch: 1.16/4.0 (29.1% complete)    lr: 0.001023    \n",
      "2020-03-18 17:03:20,756 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 528/1265    Epoch: 1.17/4.0 (29.2% complete)    lr: 0.001021    \n",
      "2020-03-18 17:03:54,422 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 529/1265    Epoch: 1.17/4.0 (29.3% complete)    lr: 0.001019    \n",
      "2020-03-18 17:04:26,684 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 530/1265    Epoch: 1.17/4.0 (29.3% complete)    lr: 0.001018    \n",
      "2020-03-18 17:05:02,432 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 531/1265    Epoch: 1.18/4.0 (29.4% complete)    lr: 0.001016    \n",
      "2020-03-18 17:05:38,874 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 532/1265    Epoch: 1.18/4.0 (29.5% complete)    lr: 0.001014    \n",
      "2020-03-18 17:06:13,240 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 533/1265    Epoch: 1.18/4.0 (29.6% complete)    lr: 0.001012    \n",
      "2020-03-18 17:06:47,169 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 534/1265    Epoch: 1.19/4.0 (29.7% complete)    lr: 0.001010    \n",
      "2020-03-18 17:07:22,226 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 535/1265    Epoch: 1.19/4.0 (29.7% complete)    lr: 0.001008    \n",
      "2020-03-18 17:07:57,337 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 536/1265    Epoch: 1.19/4.0 (29.8% complete)    lr: 0.001007    \n",
      "2020-03-18 17:08:29,682 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 537/1265    Epoch: 1.20/4.0 (29.9% complete)    lr: 0.001005    \n",
      "2020-03-18 17:09:03,226 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 538/1265    Epoch: 1.20/4.0 (30.0% complete)    lr: 0.001003    \n",
      "2020-03-18 17:09:35,529 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 539/1265    Epoch: 1.20/4.0 (30.1% complete)    lr: 0.001001    \n",
      "2020-03-18 17:10:09,038 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 540/1265    Epoch: 1.21/4.0 (30.1% complete)    lr: 0.000999    \n",
      "2020-03-18 17:10:44,768 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 541/1265    Epoch: 1.21/4.0 (30.2% complete)    lr: 0.000997    \n",
      "2020-03-18 17:11:17,179 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 542/1265    Epoch: 1.21/4.0 (30.3% complete)    lr: 0.000996    \n",
      "2020-03-18 17:11:51,107 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 543/1265    Epoch: 1.21/4.0 (30.4% complete)    lr: 0.000994    \n",
      "2020-03-18 17:12:26,972 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 544/1265    Epoch: 1.22/4.0 (30.5% complete)    lr: 0.000992    \n",
      "2020-03-18 17:13:02,858 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 545/1265    Epoch: 1.22/4.0 (30.5% complete)    lr: 0.000990    \n",
      "2020-03-18 17:13:37,976 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 546/1265    Epoch: 1.22/4.0 (30.6% complete)    lr: 0.000988    \n",
      "2020-03-18 17:14:12,725 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 547/1265    Epoch: 1.23/4.0 (30.7% complete)    lr: 0.000987    \n",
      "2020-03-18 17:14:47,431 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 548/1265    Epoch: 1.23/4.0 (30.8% complete)    lr: 0.000985    \n",
      "2020-03-18 17:15:21,976 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 549/1265    Epoch: 1.23/4.0 (30.8% complete)    lr: 0.000983    \n",
      "2020-03-18 17:15:56,470 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 550/1265    Epoch: 1.24/4.0 (30.9% complete)    lr: 0.000981    \n",
      "2020-03-18 17:16:29,846 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 551/1265    Epoch: 1.24/4.0 (31.0% complete)    lr: 0.000979    \n",
      "2020-03-18 17:17:03,026 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 552/1265    Epoch: 1.24/4.0 (31.1% complete)    lr: 0.000978    \n",
      "2020-03-18 17:17:36,103 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 553/1265    Epoch: 1.25/4.0 (31.2% complete)    lr: 0.000976    \n",
      "2020-03-18 17:18:08,289 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 554/1265    Epoch: 1.25/4.0 (31.2% complete)    lr: 0.000974    \n",
      "2020-03-18 17:18:40,450 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 555/1265    Epoch: 1.25/4.0 (31.3% complete)    lr: 0.000972    \n",
      "2020-03-18 17:19:12,072 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 556/1265    Epoch: 1.26/4.0 (31.4% complete)    lr: 0.000971    \n",
      "2020-03-18 17:19:42,889 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 557/1265    Epoch: 1.26/4.0 (31.5% complete)    lr: 0.000969    \n",
      "2020-03-18 17:20:17,147 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 558/1265    Epoch: 1.26/4.0 (31.6% complete)    lr: 0.000967    \n",
      "2020-03-18 17:20:52,377 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 559/1265    Epoch: 1.27/4.0 (31.6% complete)    lr: 0.000965    \n",
      "2020-03-18 17:21:23,538 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 560/1265    Epoch: 1.27/4.0 (31.7% complete)    lr: 0.000964    \n",
      "2020-03-18 17:21:59,670 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 561/1265    Epoch: 1.27/4.0 (31.8% complete)    lr: 0.000962    \n",
      "2020-03-18 17:22:35,648 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 562/1265    Epoch: 1.27/4.0 (31.9% complete)    lr: 0.000960    \n",
      "2020-03-18 17:23:12,390 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 563/1265    Epoch: 1.28/4.0 (32.0% complete)    lr: 0.000958    \n",
      "2020-03-18 17:23:45,361 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 564/1265    Epoch: 1.28/4.0 (32.0% complete)    lr: 0.000957    \n",
      "2020-03-18 17:24:18,794 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 565/1265    Epoch: 1.28/4.0 (32.1% complete)    lr: 0.000955    \n",
      "2020-03-18 17:24:54,590 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 566/1265    Epoch: 1.29/4.0 (32.2% complete)    lr: 0.000953    \n",
      "2020-03-18 17:25:26,760 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 567/1265    Epoch: 1.29/4.0 (32.3% complete)    lr: 0.000951    \n",
      "2020-03-18 17:26:00,443 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 568/1265    Epoch: 1.29/4.0 (32.3% complete)    lr: 0.000950    \n",
      "2020-03-18 17:26:34,276 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 569/1265    Epoch: 1.30/4.0 (32.4% complete)    lr: 0.000948    \n",
      "2020-03-18 17:27:08,194 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 570/1265    Epoch: 1.30/4.0 (32.5% complete)    lr: 0.000946    \n",
      "2020-03-18 17:27:41,376 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 571/1265    Epoch: 1.30/4.0 (32.6% complete)    lr: 0.000944    \n",
      "2020-03-18 17:28:14,667 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 572/1265    Epoch: 1.31/4.0 (32.7% complete)    lr: 0.000943    \n",
      "2020-03-18 17:28:49,107 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 573/1265    Epoch: 1.31/4.0 (32.7% complete)    lr: 0.000941    \n",
      "2020-03-18 17:29:23,970 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 574/1265    Epoch: 1.31/4.0 (32.8% complete)    lr: 0.000939    \n",
      "2020-03-18 17:29:59,639 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 575/1265    Epoch: 1.32/4.0 (32.9% complete)    lr: 0.000938    \n",
      "2020-03-18 17:30:35,376 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 576/1265    Epoch: 1.32/4.0 (33.0% complete)    lr: 0.000936    \n",
      "2020-03-18 17:31:10,419 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 577/1265    Epoch: 1.32/4.0 (33.1% complete)    lr: 0.000934    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 17:31:45,887 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 578/1265    Epoch: 1.33/4.0 (33.1% complete)    lr: 0.000933    \n",
      "2020-03-18 17:32:19,841 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 579/1265    Epoch: 1.33/4.0 (33.2% complete)    lr: 0.000931    \n",
      "2020-03-18 17:32:54,230 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 580/1265    Epoch: 1.33/4.0 (33.3% complete)    lr: 0.000929    \n",
      "2020-03-18 17:33:30,415 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 581/1265    Epoch: 1.33/4.0 (33.4% complete)    lr: 0.000927    \n",
      "2020-03-18 17:34:03,239 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 582/1265    Epoch: 1.34/4.0 (33.5% complete)    lr: 0.000926    \n",
      "2020-03-18 17:34:37,529 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 583/1265    Epoch: 1.34/4.0 (33.5% complete)    lr: 0.000924    \n",
      "2020-03-18 17:35:09,988 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 584/1265    Epoch: 1.34/4.0 (33.6% complete)    lr: 0.000922    \n",
      "2020-03-18 17:35:43,493 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 585/1265    Epoch: 1.35/4.0 (33.7% complete)    lr: 0.000921    \n",
      "2020-03-18 17:36:15,023 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 586/1265    Epoch: 1.35/4.0 (33.8% complete)    lr: 0.000919    \n",
      "2020-03-18 17:36:48,349 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 587/1265    Epoch: 1.35/4.0 (33.8% complete)    lr: 0.000917    \n",
      "2020-03-18 17:37:25,377 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 588/1265    Epoch: 1.36/4.0 (33.9% complete)    lr: 0.000916    \n",
      "2020-03-18 17:38:09,961 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 589/1265    Epoch: 1.36/4.0 (34.0% complete)    lr: 0.000914    \n",
      "2020-03-18 17:38:50,418 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 590/1265    Epoch: 1.36/4.0 (34.1% complete)    lr: 0.000912    \n",
      "2020-03-18 17:39:28,200 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 591/1265    Epoch: 1.37/4.0 (34.2% complete)    lr: 0.000911    \n",
      "2020-03-18 17:40:10,653 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 592/1265    Epoch: 1.37/4.0 (34.2% complete)    lr: 0.000909    \n",
      "2020-03-18 17:40:55,500 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 593/1265    Epoch: 1.37/4.0 (34.3% complete)    lr: 0.000907    \n",
      "2020-03-18 17:41:41,079 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 594/1265    Epoch: 1.38/4.0 (34.4% complete)    lr: 0.000906    \n",
      "2020-03-18 17:42:21,144 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 595/1265    Epoch: 1.38/4.0 (34.5% complete)    lr: 0.000904    \n",
      "2020-03-18 17:43:04,417 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 596/1265    Epoch: 1.38/4.0 (34.6% complete)    lr: 0.000903    \n",
      "2020-03-18 17:43:45,316 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 597/1265    Epoch: 1.39/4.0 (34.6% complete)    lr: 0.000901    \n",
      "2020-03-18 17:44:25,379 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 598/1265    Epoch: 1.39/4.0 (34.7% complete)    lr: 0.000899    \n",
      "2020-03-18 17:45:14,247 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 599/1265    Epoch: 1.39/4.0 (34.8% complete)    lr: 0.000898    \n",
      "2020-03-18 17:45:55,597 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 600/1265    Epoch: 1.39/4.0 (34.9% complete)    lr: 0.000896    \n",
      "2020-03-18 17:46:44,383 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 601/1265    Epoch: 1.40/4.0 (35.0% complete)    lr: 0.000894    \n",
      "2020-03-18 17:47:31,585 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 602/1265    Epoch: 1.40/4.0 (35.0% complete)    lr: 0.000893    \n",
      "2020-03-18 17:48:16,081 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 603/1265    Epoch: 1.40/4.0 (35.1% complete)    lr: 0.000891    \n",
      "2020-03-18 17:49:00,517 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 604/1265    Epoch: 1.41/4.0 (35.2% complete)    lr: 0.000889    \n",
      "2020-03-18 17:49:40,625 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 605/1265    Epoch: 1.41/4.0 (35.3% complete)    lr: 0.000888    \n",
      "2020-03-18 17:50:21,420 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 606/1265    Epoch: 1.41/4.0 (35.3% complete)    lr: 0.000886    \n",
      "2020-03-18 17:51:03,150 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 607/1265    Epoch: 1.42/4.0 (35.4% complete)    lr: 0.000885    \n",
      "2020-03-18 17:51:45,299 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 608/1265    Epoch: 1.42/4.0 (35.5% complete)    lr: 0.000883    \n",
      "2020-03-18 17:52:28,677 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 609/1265    Epoch: 1.42/4.0 (35.6% complete)    lr: 0.000881    \n",
      "2020-03-18 17:53:09,728 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 610/1265    Epoch: 1.43/4.0 (35.7% complete)    lr: 0.000880    \n",
      "2020-03-18 17:53:59,834 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 611/1265    Epoch: 1.43/4.0 (35.7% complete)    lr: 0.000878    \n",
      "2020-03-18 17:54:47,418 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 612/1265    Epoch: 1.43/4.0 (35.8% complete)    lr: 0.000877    \n",
      "2020-03-18 17:55:35,161 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 613/1265    Epoch: 1.44/4.0 (35.9% complete)    lr: 0.000875    \n",
      "2020-03-18 17:56:14,250 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 614/1265    Epoch: 1.44/4.0 (36.0% complete)    lr: 0.000873    \n",
      "2020-03-18 17:56:51,366 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 615/1265    Epoch: 1.44/4.0 (36.1% complete)    lr: 0.000872    \n",
      "2020-03-18 17:57:31,889 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 616/1265    Epoch: 1.45/4.0 (36.1% complete)    lr: 0.000870    \n",
      "2020-03-18 17:58:10,597 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 617/1265    Epoch: 1.45/4.0 (36.2% complete)    lr: 0.000869    \n",
      "2020-03-18 17:58:49,176 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 618/1265    Epoch: 1.45/4.0 (36.3% complete)    lr: 0.000867    \n",
      "2020-03-18 17:59:29,982 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 619/1265    Epoch: 1.45/4.0 (36.4% complete)    lr: 0.000866    \n",
      "2020-03-18 18:00:07,525 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 620/1265    Epoch: 1.46/4.0 (36.5% complete)    lr: 0.000864    \n",
      "2020-03-18 18:00:47,890 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 621/1265    Epoch: 1.46/4.0 (36.5% complete)    lr: 0.000862    \n",
      "2020-03-18 18:01:23,737 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 622/1265    Epoch: 1.46/4.0 (36.6% complete)    lr: 0.000861    \n",
      "2020-03-18 18:02:01,850 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 623/1265    Epoch: 1.47/4.0 (36.7% complete)    lr: 0.000859    \n",
      "2020-03-18 18:02:54,527 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 624/1265    Epoch: 1.47/4.0 (36.8% complete)    lr: 0.000858    \n",
      "2020-03-18 18:03:40,847 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 625/1265    Epoch: 1.47/4.0 (36.8% complete)    lr: 0.000856    \n",
      "2020-03-18 18:04:31,480 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 626/1265    Epoch: 1.48/4.0 (36.9% complete)    lr: 0.000855    \n",
      "2020-03-18 18:05:07,246 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 627/1265    Epoch: 1.48/4.0 (37.0% complete)    lr: 0.000853    \n",
      "2020-03-18 18:05:52,268 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 628/1265    Epoch: 1.48/4.0 (37.1% complete)    lr: 0.000851    \n",
      "2020-03-18 18:06:41,431 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 629/1265    Epoch: 1.49/4.0 (37.2% complete)    lr: 0.000850    \n",
      "2020-03-18 18:07:33,094 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 630/1265    Epoch: 1.49/4.0 (37.2% complete)    lr: 0.000848    \n",
      "2020-03-18 18:08:20,694 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 631/1265    Epoch: 1.49/4.0 (37.3% complete)    lr: 0.000847    \n",
      "2020-03-18 18:08:59,495 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 632/1265    Epoch: 1.50/4.0 (37.4% complete)    lr: 0.000845    \n",
      "2020-03-18 18:09:47,709 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 633/1265    Epoch: 1.50/4.0 (37.5% complete)    lr: 0.000844    \n",
      "2020-03-18 18:10:39,278 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 634/1265    Epoch: 1.50/4.0 (37.6% complete)    lr: 0.000842    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 18:11:31,389 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 635/1265    Epoch: 1.51/4.0 (37.6% complete)    lr: 0.000841    \n",
      "2020-03-18 18:12:20,341 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 636/1265    Epoch: 1.51/4.0 (37.7% complete)    lr: 0.000839    \n",
      "2020-03-18 18:13:08,491 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 637/1265    Epoch: 1.51/4.0 (37.8% complete)    lr: 0.000838    \n",
      "2020-03-18 18:13:53,716 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 638/1265    Epoch: 1.52/4.0 (37.9% complete)    lr: 0.000836    \n",
      "2020-03-18 18:14:31,411 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 639/1265    Epoch: 1.52/4.0 (38.0% complete)    lr: 0.000835    \n",
      "2020-03-18 18:15:05,883 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 640/1265    Epoch: 1.52/4.0 (38.0% complete)    lr: 0.000833    \n",
      "2020-03-18 18:15:51,833 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 641/1265    Epoch: 1.52/4.0 (38.1% complete)    lr: 0.000832    \n",
      "2020-03-18 18:16:42,335 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 642/1265    Epoch: 1.53/4.0 (38.2% complete)    lr: 0.000830    \n",
      "2020-03-18 18:17:20,285 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 643/1265    Epoch: 1.53/4.0 (38.3% complete)    lr: 0.000829    \n",
      "2020-03-18 18:18:12,264 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 644/1265    Epoch: 1.53/4.0 (38.3% complete)    lr: 0.000827    \n",
      "2020-03-18 18:18:57,352 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 645/1265    Epoch: 1.54/4.0 (38.4% complete)    lr: 0.000826    \n",
      "2020-03-18 18:19:37,812 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 646/1265    Epoch: 1.54/4.0 (38.5% complete)    lr: 0.000824    \n",
      "2020-03-18 18:20:26,903 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 647/1265    Epoch: 1.54/4.0 (38.6% complete)    lr: 0.000823    \n",
      "2020-03-18 18:21:04,140 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 648/1265    Epoch: 1.55/4.0 (38.7% complete)    lr: 0.000821    \n",
      "2020-03-18 18:21:37,763 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 649/1265    Epoch: 1.55/4.0 (38.7% complete)    lr: 0.000820    \n",
      "2020-03-18 18:22:29,233 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 650/1265    Epoch: 1.55/4.0 (38.8% complete)    lr: 0.000818    \n",
      "2020-03-18 18:23:21,119 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 651/1265    Epoch: 1.56/4.0 (38.9% complete)    lr: 0.000817    \n",
      "2020-03-18 18:23:59,652 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 652/1265    Epoch: 1.56/4.0 (39.0% complete)    lr: 0.000815    \n",
      "2020-03-18 18:24:34,897 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 653/1265    Epoch: 1.56/4.0 (39.1% complete)    lr: 0.000814    \n",
      "2020-03-18 18:25:14,449 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 654/1265    Epoch: 1.57/4.0 (39.1% complete)    lr: 0.000812    \n",
      "2020-03-18 18:25:55,901 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 655/1265    Epoch: 1.57/4.0 (39.2% complete)    lr: 0.000811    \n",
      "2020-03-18 18:26:34,975 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 656/1265    Epoch: 1.57/4.0 (39.3% complete)    lr: 0.000809    \n",
      "2020-03-18 18:27:10,151 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 657/1265    Epoch: 1.58/4.0 (39.4% complete)    lr: 0.000808    \n",
      "2020-03-18 18:27:46,065 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 658/1265    Epoch: 1.58/4.0 (39.5% complete)    lr: 0.000806    \n",
      "2020-03-18 18:28:40,241 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 659/1265    Epoch: 1.58/4.0 (39.5% complete)    lr: 0.000805    \n",
      "2020-03-18 18:29:36,119 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 660/1265    Epoch: 1.58/4.0 (39.6% complete)    lr: 0.000803    \n",
      "2020-03-18 18:30:25,615 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 661/1265    Epoch: 1.59/4.0 (39.7% complete)    lr: 0.000802    \n",
      "2020-03-18 18:31:02,844 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 662/1265    Epoch: 1.59/4.0 (39.8% complete)    lr: 0.000800    \n",
      "2020-03-18 18:32:00,618 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 663/1265    Epoch: 1.59/4.0 (39.8% complete)    lr: 0.000799    \n",
      "2020-03-18 18:32:57,454 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 664/1265    Epoch: 1.60/4.0 (39.9% complete)    lr: 0.000798    \n",
      "2020-03-18 18:33:42,433 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 665/1265    Epoch: 1.60/4.0 (40.0% complete)    lr: 0.000796    \n",
      "2020-03-18 18:34:19,324 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 666/1265    Epoch: 1.60/4.0 (40.1% complete)    lr: 0.000795    \n",
      "2020-03-18 18:34:54,613 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 667/1265    Epoch: 1.61/4.0 (40.2% complete)    lr: 0.000793    \n",
      "2020-03-18 18:35:43,957 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 668/1265    Epoch: 1.61/4.0 (40.2% complete)    lr: 0.000792    \n",
      "2020-03-18 18:36:37,901 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 669/1265    Epoch: 1.61/4.0 (40.3% complete)    lr: 0.000790    \n",
      "2020-03-18 18:37:34,858 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 670/1265    Epoch: 1.62/4.0 (40.4% complete)    lr: 0.000789    \n",
      "2020-03-18 18:38:22,473 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 671/1265    Epoch: 1.62/4.0 (40.5% complete)    lr: 0.000787    \n",
      "2020-03-18 18:38:58,719 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 672/1265    Epoch: 1.62/4.0 (40.6% complete)    lr: 0.000786    \n",
      "2020-03-18 18:39:38,400 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 673/1265    Epoch: 1.63/4.0 (40.6% complete)    lr: 0.000785    \n",
      "2020-03-18 18:40:27,532 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 674/1265    Epoch: 1.63/4.0 (40.7% complete)    lr: 0.000783    \n",
      "2020-03-18 18:41:16,183 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 675/1265    Epoch: 1.63/4.0 (40.8% complete)    lr: 0.000782    \n",
      "2020-03-18 18:41:56,808 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 676/1265    Epoch: 1.64/4.0 (40.9% complete)    lr: 0.000780    \n",
      "2020-03-18 18:42:31,756 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 677/1265    Epoch: 1.64/4.0 (41.0% complete)    lr: 0.000779    \n",
      "2020-03-18 18:43:06,349 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 678/1265    Epoch: 1.64/4.0 (41.0% complete)    lr: 0.000777    \n",
      "2020-03-18 18:43:40,871 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 679/1265    Epoch: 1.64/4.0 (41.1% complete)    lr: 0.000776    \n",
      "2020-03-18 18:44:22,721 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 680/1265    Epoch: 1.65/4.0 (41.2% complete)    lr: 0.000775    \n",
      "2020-03-18 18:45:24,475 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 681/1265    Epoch: 1.65/4.0 (41.3% complete)    lr: 0.000773    \n",
      "2020-03-18 18:46:24,435 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 682/1265    Epoch: 1.65/4.0 (41.4% complete)    lr: 0.000772    \n",
      "2020-03-18 18:47:13,010 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 683/1265    Epoch: 1.66/4.0 (41.4% complete)    lr: 0.000770    \n",
      "2020-03-18 18:47:47,633 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 684/1265    Epoch: 1.66/4.0 (41.5% complete)    lr: 0.000769    \n",
      "2020-03-18 18:48:21,534 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 685/1265    Epoch: 1.66/4.0 (41.6% complete)    lr: 0.000768    \n",
      "2020-03-18 18:49:07,239 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 686/1265    Epoch: 1.67/4.0 (41.7% complete)    lr: 0.000766    \n",
      "2020-03-18 18:49:49,797 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 687/1265    Epoch: 1.67/4.0 (41.7% complete)    lr: 0.000765    \n",
      "2020-03-18 18:50:26,455 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 688/1265    Epoch: 1.67/4.0 (41.8% complete)    lr: 0.000763    \n",
      "2020-03-18 18:51:03,374 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 689/1265    Epoch: 1.68/4.0 (41.9% complete)    lr: 0.000762    \n",
      "2020-03-18 18:51:39,166 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 690/1265    Epoch: 1.68/4.0 (42.0% complete)    lr: 0.000761    \n",
      "2020-03-18 18:52:13,345 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 691/1265    Epoch: 1.68/4.0 (42.1% complete)    lr: 0.000759    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 18:52:49,461 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 692/1265    Epoch: 1.69/4.0 (42.1% complete)    lr: 0.000758    \n",
      "2020-03-18 18:53:28,239 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 693/1265    Epoch: 1.69/4.0 (42.2% complete)    lr: 0.000757    \n",
      "2020-03-18 18:54:21,205 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 694/1265    Epoch: 1.69/4.0 (42.3% complete)    lr: 0.000755    \n",
      "2020-03-18 18:55:11,474 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 695/1265    Epoch: 1.70/4.0 (42.4% complete)    lr: 0.000754    \n",
      "2020-03-18 18:55:58,540 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 696/1265    Epoch: 1.70/4.0 (42.5% complete)    lr: 0.000752    \n",
      "2020-03-18 18:56:37,364 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 697/1265    Epoch: 1.70/4.0 (42.5% complete)    lr: 0.000751    \n",
      "2020-03-18 18:57:15,441 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 698/1265    Epoch: 1.70/4.0 (42.6% complete)    lr: 0.000750    \n",
      "2020-03-18 18:58:04,648 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 699/1265    Epoch: 1.71/4.0 (42.7% complete)    lr: 0.000748    \n",
      "2020-03-18 18:58:57,854 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 700/1265    Epoch: 1.71/4.0 (42.8% complete)    lr: 0.000747    \n",
      "2020-03-18 18:59:43,646 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 701/1265    Epoch: 1.71/4.0 (42.9% complete)    lr: 0.000746    \n",
      "2020-03-18 19:00:17,065 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 702/1265    Epoch: 1.72/4.0 (42.9% complete)    lr: 0.000744    \n",
      "2020-03-18 19:00:52,373 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 703/1265    Epoch: 1.72/4.0 (43.0% complete)    lr: 0.000743    \n",
      "2020-03-18 19:01:26,718 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 704/1265    Epoch: 1.72/4.0 (43.1% complete)    lr: 0.000742    \n",
      "2020-03-18 19:02:02,483 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 705/1265    Epoch: 1.73/4.0 (43.2% complete)    lr: 0.000740    \n",
      "2020-03-18 19:02:36,920 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 706/1265    Epoch: 1.73/4.0 (43.2% complete)    lr: 0.000739    \n",
      "2020-03-18 19:03:25,866 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 707/1265    Epoch: 1.73/4.0 (43.3% complete)    lr: 0.000738    \n",
      "2020-03-18 19:04:16,893 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 708/1265    Epoch: 1.74/4.0 (43.4% complete)    lr: 0.000736    \n",
      "2020-03-18 19:05:00,130 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 709/1265    Epoch: 1.74/4.0 (43.5% complete)    lr: 0.000735    \n",
      "2020-03-18 19:05:36,094 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 710/1265    Epoch: 1.74/4.0 (43.6% complete)    lr: 0.000734    \n",
      "2020-03-18 19:06:12,898 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 711/1265    Epoch: 1.75/4.0 (43.6% complete)    lr: 0.000732    \n",
      "2020-03-18 19:06:48,582 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 712/1265    Epoch: 1.75/4.0 (43.7% complete)    lr: 0.000731    \n",
      "2020-03-18 19:07:24,124 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 713/1265    Epoch: 1.75/4.0 (43.8% complete)    lr: 0.000730    \n",
      "2020-03-18 19:08:00,684 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 714/1265    Epoch: 1.76/4.0 (43.9% complete)    lr: 0.000728    \n",
      "2020-03-18 19:08:40,432 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 715/1265    Epoch: 1.76/4.0 (44.0% complete)    lr: 0.000727    \n",
      "2020-03-18 19:09:38,346 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 716/1265    Epoch: 1.76/4.0 (44.0% complete)    lr: 0.000726    \n",
      "2020-03-18 19:10:34,392 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 717/1265    Epoch: 1.76/4.0 (44.1% complete)    lr: 0.000724    \n",
      "2020-03-18 19:11:23,186 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 718/1265    Epoch: 1.77/4.0 (44.2% complete)    lr: 0.000723    \n",
      "2020-03-18 19:11:57,199 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 719/1265    Epoch: 1.77/4.0 (44.3% complete)    lr: 0.000722    \n",
      "2020-03-18 19:12:33,969 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 720/1265    Epoch: 1.77/4.0 (44.4% complete)    lr: 0.000720    \n",
      "2020-03-18 19:13:29,832 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 721/1265    Epoch: 1.78/4.0 (44.4% complete)    lr: 0.000719    \n",
      "2020-03-18 19:14:17,192 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 722/1265    Epoch: 1.78/4.0 (44.5% complete)    lr: 0.000718    \n",
      "2020-03-18 19:14:58,564 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 723/1265    Epoch: 1.78/4.0 (44.6% complete)    lr: 0.000716    \n",
      "2020-03-18 19:15:37,696 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 724/1265    Epoch: 1.79/4.0 (44.7% complete)    lr: 0.000715    \n",
      "2020-03-18 19:16:09,795 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 725/1265    Epoch: 1.79/4.0 (44.7% complete)    lr: 0.000714    \n",
      "2020-03-18 19:16:44,777 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 726/1265    Epoch: 1.79/4.0 (44.8% complete)    lr: 0.000712    \n",
      "2020-03-18 19:17:22,284 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 727/1265    Epoch: 1.80/4.0 (44.9% complete)    lr: 0.000711    \n",
      "2020-03-18 19:18:00,617 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 728/1265    Epoch: 1.80/4.0 (45.0% complete)    lr: 0.000710    \n",
      "2020-03-18 19:18:47,206 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 729/1265    Epoch: 1.80/4.0 (45.1% complete)    lr: 0.000709    \n",
      "2020-03-18 19:19:39,834 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 730/1265    Epoch: 1.81/4.0 (45.1% complete)    lr: 0.000707    \n",
      "2020-03-18 19:20:35,585 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 731/1265    Epoch: 1.81/4.0 (45.2% complete)    lr: 0.000706    \n",
      "2020-03-18 19:21:14,705 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 732/1265    Epoch: 1.81/4.0 (45.3% complete)    lr: 0.000705    \n",
      "2020-03-18 19:21:50,230 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 733/1265    Epoch: 1.82/4.0 (45.4% complete)    lr: 0.000703    \n",
      "2020-03-18 19:22:40,523 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 734/1265    Epoch: 1.82/4.0 (45.5% complete)    lr: 0.000702    \n",
      "2020-03-18 19:23:32,041 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 735/1265    Epoch: 1.82/4.0 (45.5% complete)    lr: 0.000701    \n",
      "2020-03-18 19:24:22,621 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 736/1265    Epoch: 1.82/4.0 (45.6% complete)    lr: 0.000700    \n",
      "2020-03-18 19:24:59,713 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 737/1265    Epoch: 1.83/4.0 (45.7% complete)    lr: 0.000698    \n",
      "2020-03-18 19:25:32,749 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 738/1265    Epoch: 1.83/4.0 (45.8% complete)    lr: 0.000697    \n",
      "2020-03-18 19:26:09,652 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 739/1265    Epoch: 1.83/4.0 (45.9% complete)    lr: 0.000696    \n",
      "2020-03-18 19:26:53,385 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 740/1265    Epoch: 1.84/4.0 (45.9% complete)    lr: 0.000695    \n",
      "2020-03-18 19:27:51,501 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 741/1265    Epoch: 1.84/4.0 (46.0% complete)    lr: 0.000693    \n",
      "2020-03-18 19:28:36,090 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 742/1265    Epoch: 1.84/4.0 (46.1% complete)    lr: 0.000692    \n",
      "2020-03-18 19:29:13,220 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 743/1265    Epoch: 1.85/4.0 (46.2% complete)    lr: 0.000691    \n",
      "2020-03-18 19:30:03,032 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 744/1265    Epoch: 1.85/4.0 (46.2% complete)    lr: 0.000690    \n",
      "2020-03-18 19:30:54,471 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 745/1265    Epoch: 1.85/4.0 (46.3% complete)    lr: 0.000688    \n",
      "2020-03-18 19:31:45,709 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 746/1265    Epoch: 1.86/4.0 (46.4% complete)    lr: 0.000687    \n",
      "2020-03-18 19:32:36,908 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 747/1265    Epoch: 1.86/4.0 (46.5% complete)    lr: 0.000686    \n",
      "2020-03-18 19:33:27,216 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 748/1265    Epoch: 1.86/4.0 (46.6% complete)    lr: 0.000685    \n",
      "2020-03-18 19:34:16,809 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 749/1265    Epoch: 1.87/4.0 (46.6% complete)    lr: 0.000683    \n",
      "2020-03-18 19:35:06,097 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 750/1265    Epoch: 1.87/4.0 (46.7% complete)    lr: 0.000682    \n",
      "2020-03-18 19:35:55,037 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 751/1265    Epoch: 1.87/4.0 (46.8% complete)    lr: 0.000681    \n",
      "2020-03-18 19:36:45,846 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 752/1265    Epoch: 1.88/4.0 (46.9% complete)    lr: 0.000680    \n",
      "2020-03-18 19:37:33,303 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 753/1265    Epoch: 1.88/4.0 (47.0% complete)    lr: 0.000678    \n",
      "2020-03-18 19:38:15,047 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 754/1265    Epoch: 1.88/4.0 (47.0% complete)    lr: 0.000677    \n",
      "2020-03-18 19:38:52,421 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 755/1265    Epoch: 1.88/4.0 (47.1% complete)    lr: 0.000676    \n",
      "2020-03-18 19:39:28,703 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 756/1265    Epoch: 1.89/4.0 (47.2% complete)    lr: 0.000675    \n",
      "2020-03-18 19:40:04,812 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 757/1265    Epoch: 1.89/4.0 (47.3% complete)    lr: 0.000673    \n",
      "2020-03-18 19:40:46,011 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 758/1265    Epoch: 1.89/4.0 (47.4% complete)    lr: 0.000672    \n",
      "2020-03-18 19:41:26,761 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 759/1265    Epoch: 1.90/4.0 (47.4% complete)    lr: 0.000671    \n",
      "2020-03-18 19:42:11,308 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 760/1265    Epoch: 1.90/4.0 (47.5% complete)    lr: 0.000670    \n",
      "2020-03-18 19:42:56,263 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 761/1265    Epoch: 1.90/4.0 (47.6% complete)    lr: 0.000669    \n",
      "2020-03-18 19:43:41,380 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 762/1265    Epoch: 1.91/4.0 (47.7% complete)    lr: 0.000667    \n",
      "2020-03-18 19:44:26,914 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 763/1265    Epoch: 1.91/4.0 (47.7% complete)    lr: 0.000666    \n",
      "2020-03-18 19:45:03,990 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 764/1265    Epoch: 1.91/4.0 (47.8% complete)    lr: 0.000665    \n",
      "2020-03-18 19:45:42,032 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 765/1265    Epoch: 1.92/4.0 (47.9% complete)    lr: 0.000664    \n",
      "2020-03-18 19:46:21,687 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 766/1265    Epoch: 1.92/4.0 (48.0% complete)    lr: 0.000662    \n",
      "2020-03-18 19:46:59,565 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 767/1265    Epoch: 1.92/4.0 (48.1% complete)    lr: 0.000661    \n",
      "2020-03-18 19:47:38,512 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 768/1265    Epoch: 1.93/4.0 (48.1% complete)    lr: 0.000660    \n",
      "2020-03-18 19:48:15,269 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 769/1265    Epoch: 1.93/4.0 (48.2% complete)    lr: 0.000659    \n",
      "2020-03-18 19:48:49,389 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 770/1265    Epoch: 1.93/4.0 (48.3% complete)    lr: 0.000658    \n",
      "2020-03-18 19:49:43,675 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 771/1265    Epoch: 1.94/4.0 (48.4% complete)    lr: 0.000656    \n",
      "2020-03-18 19:50:39,498 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 772/1265    Epoch: 1.94/4.0 (48.5% complete)    lr: 0.000655    \n",
      "2020-03-18 19:51:35,408 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 773/1265    Epoch: 1.94/4.0 (48.5% complete)    lr: 0.000654    \n",
      "2020-03-18 19:52:21,147 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 774/1265    Epoch: 1.94/4.0 (48.6% complete)    lr: 0.000653    \n",
      "2020-03-18 19:52:55,635 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 775/1265    Epoch: 1.95/4.0 (48.7% complete)    lr: 0.000652    \n",
      "2020-03-18 19:53:36,907 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 776/1265    Epoch: 1.95/4.0 (48.8% complete)    lr: 0.000651    \n",
      "2020-03-18 19:54:13,721 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 777/1265    Epoch: 1.95/4.0 (48.9% complete)    lr: 0.000649    \n",
      "2020-03-18 19:54:58,853 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 778/1265    Epoch: 1.96/4.0 (48.9% complete)    lr: 0.000648    \n",
      "2020-03-18 19:55:36,035 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 779/1265    Epoch: 1.96/4.0 (49.0% complete)    lr: 0.000647    \n",
      "2020-03-18 19:56:28,787 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 780/1265    Epoch: 1.96/4.0 (49.1% complete)    lr: 0.000646    \n",
      "2020-03-18 19:57:19,720 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 781/1265    Epoch: 1.97/4.0 (49.2% complete)    lr: 0.000645    \n",
      "2020-03-18 19:58:16,835 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 782/1265    Epoch: 1.97/4.0 (49.2% complete)    lr: 0.000643    \n",
      "2020-03-18 19:59:03,593 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 783/1265    Epoch: 1.97/4.0 (49.3% complete)    lr: 0.000642    \n",
      "2020-03-18 19:59:43,901 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 784/1265    Epoch: 1.98/4.0 (49.4% complete)    lr: 0.000641    \n",
      "2020-03-18 20:00:40,672 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 785/1265    Epoch: 1.98/4.0 (49.5% complete)    lr: 0.000640    \n",
      "2020-03-18 20:01:29,129 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 786/1265    Epoch: 1.98/4.0 (49.6% complete)    lr: 0.000639    \n",
      "2020-03-18 20:02:09,420 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 787/1265    Epoch: 1.99/4.0 (49.6% complete)    lr: 0.000638    \n",
      "2020-03-18 20:02:50,208 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 788/1265    Epoch: 1.99/4.0 (49.7% complete)    lr: 0.000636    \n",
      "2020-03-18 20:03:31,625 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 789/1265    Epoch: 1.99/4.0 (49.8% complete)    lr: 0.000635    \n",
      "2020-03-18 20:04:22,098 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 790/1265    Epoch: 2.00/4.0 (49.9% complete)    lr: 0.000634    \n",
      "2020-03-18 20:05:11,832 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 791/1265    Epoch: 2.00/4.0 (50.0% complete)    lr: 0.000633    \n",
      "2020-03-18 20:05:54,063 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 792/1265    Epoch: 2.00/4.0 (50.0% complete)    lr: 0.000632    \n",
      "2020-03-18 20:06:30,909 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 793/1265    Epoch: 2.00/4.0 (50.1% complete)    lr: 0.000631    \n",
      "2020-03-18 20:07:29,026 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 794/1265    Epoch: 2.01/4.0 (50.2% complete)    lr: 0.000630    \n",
      "2020-03-18 20:08:29,146 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 795/1265    Epoch: 2.01/4.0 (50.3% complete)    lr: 0.000628    \n",
      "2020-03-18 20:09:24,239 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 796/1265    Epoch: 2.01/4.0 (50.4% complete)    lr: 0.000627    \n",
      "2020-03-18 20:10:03,826 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 797/1265    Epoch: 2.02/4.0 (50.4% complete)    lr: 0.000626    \n",
      "2020-03-18 20:10:59,342 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 798/1265    Epoch: 2.02/4.0 (50.5% complete)    lr: 0.000625    \n",
      "2020-03-18 20:11:57,038 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 799/1265    Epoch: 2.02/4.0 (50.6% complete)    lr: 0.000624    \n",
      "2020-03-18 20:12:40,926 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 800/1265    Epoch: 2.03/4.0 (50.7% complete)    lr: 0.000623    \n",
      "2020-03-18 20:13:20,896 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 801/1265    Epoch: 2.03/4.0 (50.8% complete)    lr: 0.000622    \n",
      "2020-03-18 20:13:59,944 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 802/1265    Epoch: 2.03/4.0 (50.8% complete)    lr: 0.000620    \n",
      "2020-03-18 20:14:36,533 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 803/1265    Epoch: 2.04/4.0 (50.9% complete)    lr: 0.000619    \n",
      "2020-03-18 20:15:14,388 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 804/1265    Epoch: 2.04/4.0 (51.0% complete)    lr: 0.000618    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 20:15:48,357 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 805/1265    Epoch: 2.04/4.0 (51.1% complete)    lr: 0.000617    \n",
      "2020-03-18 20:16:25,333 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 806/1265    Epoch: 2.05/4.0 (51.1% complete)    lr: 0.000616    \n",
      "2020-03-18 20:17:06,827 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 807/1265    Epoch: 2.05/4.0 (51.2% complete)    lr: 0.000615    \n",
      "2020-03-18 20:17:49,162 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 808/1265    Epoch: 2.05/4.0 (51.3% complete)    lr: 0.000614    \n",
      "2020-03-18 20:18:51,562 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 809/1265    Epoch: 2.06/4.0 (51.4% complete)    lr: 0.000613    \n",
      "2020-03-18 20:19:50,273 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 810/1265    Epoch: 2.06/4.0 (51.5% complete)    lr: 0.000612    \n",
      "2020-03-18 20:20:48,680 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 811/1265    Epoch: 2.06/4.0 (51.5% complete)    lr: 0.000610    \n",
      "2020-03-18 20:21:35,592 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 812/1265    Epoch: 2.06/4.0 (51.6% complete)    lr: 0.000609    \n",
      "2020-03-18 20:22:18,915 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 813/1265    Epoch: 2.07/4.0 (51.7% complete)    lr: 0.000608    \n",
      "2020-03-18 20:23:21,679 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 814/1265    Epoch: 2.07/4.0 (51.8% complete)    lr: 0.000607    \n",
      "2020-03-18 20:24:23,863 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 815/1265    Epoch: 2.07/4.0 (51.9% complete)    lr: 0.000606    \n",
      "2020-03-18 20:25:19,208 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 816/1265    Epoch: 2.08/4.0 (51.9% complete)    lr: 0.000605    \n",
      "2020-03-18 20:26:08,408 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 817/1265    Epoch: 2.08/4.0 (52.0% complete)    lr: 0.000604    \n",
      "2020-03-18 20:26:59,991 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 818/1265    Epoch: 2.08/4.0 (52.1% complete)    lr: 0.000603    \n",
      "2020-03-18 20:27:39,150 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 819/1265    Epoch: 2.09/4.0 (52.2% complete)    lr: 0.000602    \n",
      "2020-03-18 20:28:20,051 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 820/1265    Epoch: 2.09/4.0 (52.3% complete)    lr: 0.000601    \n",
      "2020-03-18 20:28:59,708 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 821/1265    Epoch: 2.09/4.0 (52.3% complete)    lr: 0.000599    \n",
      "2020-03-18 20:29:35,408 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 822/1265    Epoch: 2.10/4.0 (52.4% complete)    lr: 0.000598    \n",
      "2020-03-18 20:30:16,816 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 823/1265    Epoch: 2.10/4.0 (52.5% complete)    lr: 0.000597    \n",
      "2020-03-18 20:31:15,294 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 824/1265    Epoch: 2.10/4.0 (52.6% complete)    lr: 0.000596    \n",
      "2020-03-18 20:32:12,012 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 825/1265    Epoch: 2.11/4.0 (52.6% complete)    lr: 0.000595    \n",
      "2020-03-18 20:33:09,724 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 826/1265    Epoch: 2.11/4.0 (52.7% complete)    lr: 0.000594    \n",
      "2020-03-18 20:34:11,146 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 827/1265    Epoch: 2.11/4.0 (52.8% complete)    lr: 0.000593    \n",
      "2020-03-18 20:35:07,172 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 828/1265    Epoch: 2.12/4.0 (52.9% complete)    lr: 0.000592    \n",
      "2020-03-18 20:35:54,420 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 829/1265    Epoch: 2.12/4.0 (53.0% complete)    lr: 0.000591    \n",
      "2020-03-18 20:36:33,370 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 830/1265    Epoch: 2.12/4.0 (53.0% complete)    lr: 0.000590    \n",
      "2020-03-18 20:37:15,317 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 831/1265    Epoch: 2.12/4.0 (53.1% complete)    lr: 0.000589    \n",
      "2020-03-18 20:37:53,106 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 832/1265    Epoch: 2.13/4.0 (53.2% complete)    lr: 0.000588    \n",
      "2020-03-18 20:38:31,580 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 833/1265    Epoch: 2.13/4.0 (53.3% complete)    lr: 0.000586    \n",
      "2020-03-18 20:39:27,327 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 834/1265    Epoch: 2.13/4.0 (53.4% complete)    lr: 0.000585    \n",
      "2020-03-18 20:40:22,003 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 835/1265    Epoch: 2.14/4.0 (53.4% complete)    lr: 0.000584    \n",
      "2020-03-18 20:41:17,382 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 836/1265    Epoch: 2.14/4.0 (53.5% complete)    lr: 0.000583    \n",
      "2020-03-18 20:42:12,248 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 837/1265    Epoch: 2.14/4.0 (53.6% complete)    lr: 0.000582    \n",
      "2020-03-18 20:43:05,820 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 838/1265    Epoch: 2.15/4.0 (53.7% complete)    lr: 0.000581    \n",
      "2020-03-18 20:43:58,893 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 839/1265    Epoch: 2.15/4.0 (53.8% complete)    lr: 0.000580    \n",
      "2020-03-18 20:44:50,911 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 840/1265    Epoch: 2.15/4.0 (53.8% complete)    lr: 0.000579    \n",
      "2020-03-18 20:45:34,720 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 841/1265    Epoch: 2.16/4.0 (53.9% complete)    lr: 0.000578    \n",
      "2020-03-18 20:46:14,637 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 842/1265    Epoch: 2.16/4.0 (54.0% complete)    lr: 0.000577    \n",
      "2020-03-18 20:46:54,670 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 843/1265    Epoch: 2.16/4.0 (54.1% complete)    lr: 0.000576    \n",
      "2020-03-18 20:47:33,880 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 844/1265    Epoch: 2.17/4.0 (54.1% complete)    lr: 0.000575    \n",
      "2020-03-18 20:48:10,822 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 845/1265    Epoch: 2.17/4.0 (54.2% complete)    lr: 0.000574    \n",
      "2020-03-18 20:48:46,665 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 846/1265    Epoch: 2.17/4.0 (54.3% complete)    lr: 0.000573    \n",
      "2020-03-18 20:49:23,718 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 847/1265    Epoch: 2.18/4.0 (54.4% complete)    lr: 0.000572    \n",
      "2020-03-18 20:50:04,472 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 848/1265    Epoch: 2.18/4.0 (54.5% complete)    lr: 0.000571    \n",
      "2020-03-18 20:50:56,297 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 849/1265    Epoch: 2.18/4.0 (54.5% complete)    lr: 0.000570    \n",
      "2020-03-18 20:51:47,168 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 850/1265    Epoch: 2.18/4.0 (54.6% complete)    lr: 0.000569    \n",
      "2020-03-18 20:52:32,102 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 851/1265    Epoch: 2.19/4.0 (54.7% complete)    lr: 0.000568    \n",
      "2020-03-18 20:53:08,990 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 852/1265    Epoch: 2.19/4.0 (54.8% complete)    lr: 0.000567    \n",
      "2020-03-18 20:53:45,791 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 853/1265    Epoch: 2.19/4.0 (54.9% complete)    lr: 0.000566    \n",
      "2020-03-18 20:54:32,400 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 854/1265    Epoch: 2.20/4.0 (54.9% complete)    lr: 0.000564    \n",
      "2020-03-18 20:55:13,691 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 855/1265    Epoch: 2.20/4.0 (55.0% complete)    lr: 0.000563    \n",
      "2020-03-18 20:55:53,324 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 856/1265    Epoch: 2.20/4.0 (55.1% complete)    lr: 0.000562    \n",
      "2020-03-18 20:56:28,983 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 857/1265    Epoch: 2.21/4.0 (55.2% complete)    lr: 0.000561    \n",
      "2020-03-18 20:57:05,734 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 858/1265    Epoch: 2.21/4.0 (55.3% complete)    lr: 0.000560    \n",
      "2020-03-18 20:57:42,678 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 859/1265    Epoch: 2.21/4.0 (55.3% complete)    lr: 0.000559    \n",
      "2020-03-18 20:58:16,159 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 860/1265    Epoch: 2.22/4.0 (55.4% complete)    lr: 0.000558    \n",
      "2020-03-18 20:58:52,810 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 861/1265    Epoch: 2.22/4.0 (55.5% complete)    lr: 0.000557    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 20:59:28,180 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 862/1265    Epoch: 2.22/4.0 (55.6% complete)    lr: 0.000556    \n",
      "2020-03-18 21:00:15,463 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 863/1265    Epoch: 2.23/4.0 (55.6% complete)    lr: 0.000555    \n",
      "2020-03-18 21:01:06,993 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 864/1265    Epoch: 2.23/4.0 (55.7% complete)    lr: 0.000554    \n",
      "2020-03-18 21:01:59,357 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 865/1265    Epoch: 2.23/4.0 (55.8% complete)    lr: 0.000553    \n",
      "2020-03-18 21:02:57,680 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 866/1265    Epoch: 2.24/4.0 (55.9% complete)    lr: 0.000552    \n",
      "2020-03-18 21:03:54,273 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 867/1265    Epoch: 2.24/4.0 (56.0% complete)    lr: 0.000551    \n",
      "2020-03-18 21:04:50,321 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 868/1265    Epoch: 2.24/4.0 (56.0% complete)    lr: 0.000550    \n",
      "2020-03-18 21:05:45,854 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 869/1265    Epoch: 2.24/4.0 (56.1% complete)    lr: 0.000549    \n",
      "2020-03-18 21:06:35,950 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 870/1265    Epoch: 2.25/4.0 (56.2% complete)    lr: 0.000548    \n",
      "2020-03-18 21:07:11,043 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 871/1265    Epoch: 2.25/4.0 (56.3% complete)    lr: 0.000547    \n",
      "2020-03-18 21:07:45,155 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 872/1265    Epoch: 2.25/4.0 (56.4% complete)    lr: 0.000546    \n",
      "2020-03-18 21:08:38,955 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 873/1265    Epoch: 2.26/4.0 (56.4% complete)    lr: 0.000545    \n",
      "2020-03-18 21:09:37,913 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 874/1265    Epoch: 2.26/4.0 (56.5% complete)    lr: 0.000544    \n",
      "2020-03-18 21:10:35,251 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 875/1265    Epoch: 2.26/4.0 (56.6% complete)    lr: 0.000543    \n",
      "2020-03-18 21:11:29,635 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 876/1265    Epoch: 2.27/4.0 (56.7% complete)    lr: 0.000542    \n",
      "2020-03-18 21:12:24,950 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 877/1265    Epoch: 2.27/4.0 (56.8% complete)    lr: 0.000541    \n",
      "2020-03-18 21:13:17,661 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 878/1265    Epoch: 2.27/4.0 (56.8% complete)    lr: 0.000540    \n",
      "2020-03-18 21:14:13,950 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 879/1265    Epoch: 2.28/4.0 (56.9% complete)    lr: 0.000539    \n",
      "2020-03-18 21:14:52,810 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 880/1265    Epoch: 2.28/4.0 (57.0% complete)    lr: 0.000538    \n",
      "2020-03-18 21:15:29,847 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 881/1265    Epoch: 2.28/4.0 (57.1% complete)    lr: 0.000537    \n",
      "2020-03-18 21:16:03,134 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 882/1265    Epoch: 2.29/4.0 (57.1% complete)    lr: 0.000536    \n",
      "2020-03-18 21:16:36,088 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 883/1265    Epoch: 2.29/4.0 (57.2% complete)    lr: 0.000535    \n",
      "2020-03-18 21:17:13,253 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 884/1265    Epoch: 2.29/4.0 (57.3% complete)    lr: 0.000535    \n",
      "2020-03-18 21:17:50,301 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 885/1265    Epoch: 2.30/4.0 (57.4% complete)    lr: 0.000534    \n",
      "2020-03-18 21:18:29,273 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 886/1265    Epoch: 2.30/4.0 (57.5% complete)    lr: 0.000533    \n",
      "2020-03-18 21:19:08,447 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 887/1265    Epoch: 2.30/4.0 (57.5% complete)    lr: 0.000532    \n",
      "2020-03-18 21:20:05,020 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 888/1265    Epoch: 2.30/4.0 (57.6% complete)    lr: 0.000531    \n",
      "2020-03-18 21:21:03,219 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 889/1265    Epoch: 2.31/4.0 (57.7% complete)    lr: 0.000530    \n",
      "2020-03-18 21:22:01,643 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 890/1265    Epoch: 2.31/4.0 (57.8% complete)    lr: 0.000529    \n",
      "2020-03-18 21:22:54,730 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 891/1265    Epoch: 2.31/4.0 (57.9% complete)    lr: 0.000528    \n",
      "2020-03-18 21:23:33,735 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 892/1265    Epoch: 2.32/4.0 (57.9% complete)    lr: 0.000527    \n",
      "2020-03-18 21:24:27,847 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 893/1265    Epoch: 2.32/4.0 (58.0% complete)    lr: 0.000526    \n",
      "2020-03-18 21:25:20,692 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 894/1265    Epoch: 2.32/4.0 (58.1% complete)    lr: 0.000525    \n",
      "2020-03-18 21:26:11,639 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 895/1265    Epoch: 2.33/4.0 (58.2% complete)    lr: 0.000524    \n",
      "2020-03-18 21:26:50,342 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 896/1265    Epoch: 2.33/4.0 (58.3% complete)    lr: 0.000523    \n",
      "2020-03-18 21:27:23,886 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 897/1265    Epoch: 2.33/4.0 (58.3% complete)    lr: 0.000522    \n",
      "2020-03-18 21:28:18,416 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 898/1265    Epoch: 2.34/4.0 (58.4% complete)    lr: 0.000521    \n",
      "2020-03-18 21:29:18,310 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 899/1265    Epoch: 2.34/4.0 (58.5% complete)    lr: 0.000520    \n",
      "2020-03-18 21:30:14,972 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 900/1265    Epoch: 2.34/4.0 (58.6% complete)    lr: 0.000519    \n",
      "2020-03-18 21:31:11,538 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 901/1265    Epoch: 2.35/4.0 (58.6% complete)    lr: 0.000518    \n",
      "2020-03-18 21:31:49,219 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 902/1265    Epoch: 2.35/4.0 (58.7% complete)    lr: 0.000517    \n",
      "2020-03-18 21:32:42,166 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 903/1265    Epoch: 2.35/4.0 (58.8% complete)    lr: 0.000516    \n",
      "2020-03-18 21:33:40,152 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 904/1265    Epoch: 2.36/4.0 (58.9% complete)    lr: 0.000515    \n",
      "2020-03-18 21:34:39,306 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 905/1265    Epoch: 2.36/4.0 (59.0% complete)    lr: 0.000514    \n",
      "2020-03-18 21:35:41,171 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 906/1265    Epoch: 2.36/4.0 (59.0% complete)    lr: 0.000514    \n",
      "2020-03-18 21:36:34,606 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 907/1265    Epoch: 2.36/4.0 (59.1% complete)    lr: 0.000513    \n",
      "2020-03-18 21:37:18,069 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 908/1265    Epoch: 2.37/4.0 (59.2% complete)    lr: 0.000512    \n",
      "2020-03-18 21:37:56,008 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 909/1265    Epoch: 2.37/4.0 (59.3% complete)    lr: 0.000511    \n",
      "2020-03-18 21:38:34,382 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 910/1265    Epoch: 2.37/4.0 (59.4% complete)    lr: 0.000510    \n",
      "2020-03-18 21:39:34,073 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 911/1265    Epoch: 2.38/4.0 (59.4% complete)    lr: 0.000509    \n",
      "2020-03-18 21:40:20,119 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 912/1265    Epoch: 2.38/4.0 (59.5% complete)    lr: 0.000508    \n",
      "2020-03-18 21:40:56,441 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 913/1265    Epoch: 2.38/4.0 (59.6% complete)    lr: 0.000507    \n",
      "2020-03-18 21:41:34,482 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 914/1265    Epoch: 2.39/4.0 (59.7% complete)    lr: 0.000506    \n",
      "2020-03-18 21:42:19,203 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 915/1265    Epoch: 2.39/4.0 (59.8% complete)    lr: 0.000505    \n",
      "2020-03-18 21:43:03,924 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 916/1265    Epoch: 2.39/4.0 (59.8% complete)    lr: 0.000504    \n",
      "2020-03-18 21:43:41,821 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 917/1265    Epoch: 2.40/4.0 (59.9% complete)    lr: 0.000503    \n",
      "2020-03-18 21:44:19,292 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 918/1265    Epoch: 2.40/4.0 (60.0% complete)    lr: 0.000502    \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-18 21:45:04,653 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 919/1265    Epoch: 2.40/4.0 (60.1% complete)    lr: 0.000502    \n",
      "2020-03-18 21:45:50,156 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 920/1265    Epoch: 2.41/4.0 (60.2% complete)    lr: 0.000501    \n",
      "2020-03-18 21:46:38,356 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 921/1265    Epoch: 2.41/4.0 (60.2% complete)    lr: 0.000500    \n",
      "2020-03-18 21:47:17,395 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 922/1265    Epoch: 2.41/4.0 (60.3% complete)    lr: 0.000499    \n",
      "2020-03-18 21:47:55,770 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 923/1265    Epoch: 2.42/4.0 (60.4% complete)    lr: 0.000498    \n",
      "2020-03-18 21:48:32,465 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 924/1265    Epoch: 2.42/4.0 (60.5% complete)    lr: 0.000497    \n",
      "2020-03-18 21:49:10,186 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 925/1265    Epoch: 2.42/4.0 (60.5% complete)    lr: 0.000496    \n",
      "2020-03-18 21:49:46,626 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 926/1265    Epoch: 2.42/4.0 (60.6% complete)    lr: 0.000495    \n",
      "2020-03-18 21:50:24,465 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 927/1265    Epoch: 2.43/4.0 (60.7% complete)    lr: 0.000494    \n",
      "2020-03-18 21:51:00,237 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 928/1265    Epoch: 2.43/4.0 (60.8% complete)    lr: 0.000493    \n",
      "2020-03-18 21:51:46,540 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 929/1265    Epoch: 2.43/4.0 (60.9% complete)    lr: 0.000493    \n",
      "2020-03-18 21:52:42,215 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 930/1265    Epoch: 2.44/4.0 (60.9% complete)    lr: 0.000492    \n",
      "2020-03-18 21:53:32,905 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 931/1265    Epoch: 2.44/4.0 (61.0% complete)    lr: 0.000491    \n",
      "2020-03-18 21:54:12,042 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 932/1265    Epoch: 2.44/4.0 (61.1% complete)    lr: 0.000490    \n",
      "2020-03-18 21:54:47,081 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 933/1265    Epoch: 2.45/4.0 (61.2% complete)    lr: 0.000489    \n",
      "2020-03-18 21:55:23,806 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 934/1265    Epoch: 2.45/4.0 (61.3% complete)    lr: 0.000488    \n",
      "2020-03-18 21:56:01,015 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 935/1265    Epoch: 2.45/4.0 (61.3% complete)    lr: 0.000487    \n",
      "2020-03-18 21:56:57,189 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 936/1265    Epoch: 2.46/4.0 (61.4% complete)    lr: 0.000486    \n",
      "2020-03-18 21:57:30,335 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 937/1265    Epoch: 2.46/4.0 (61.5% complete)    lr: 0.000485    \n",
      "2020-03-18 21:58:17,827 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 938/1265    Epoch: 2.46/4.0 (61.6% complete)    lr: 0.000485    \n",
      "2020-03-18 21:59:03,879 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 939/1265    Epoch: 2.47/4.0 (61.7% complete)    lr: 0.000484    \n",
      "2020-03-18 21:59:44,758 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 940/1265    Epoch: 2.47/4.0 (61.7% complete)    lr: 0.000483    \n",
      "2020-03-18 22:00:39,724 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 941/1265    Epoch: 2.47/4.0 (61.8% complete)    lr: 0.000482    \n",
      "2020-03-18 22:01:12,404 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 942/1265    Epoch: 2.48/4.0 (61.9% complete)    lr: 0.000481    \n",
      "2020-03-18 22:01:58,510 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 943/1265    Epoch: 2.48/4.0 (62.0% complete)    lr: 0.000480    \n",
      "2020-03-18 22:02:52,546 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 944/1265    Epoch: 2.48/4.0 (62.0% complete)    lr: 0.000479    \n",
      "2020-03-18 22:03:39,004 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 945/1265    Epoch: 2.48/4.0 (62.1% complete)    lr: 0.000478    \n",
      "2020-03-18 22:04:19,749 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 946/1265    Epoch: 2.49/4.0 (62.2% complete)    lr: 0.000478    \n",
      "2020-03-18 22:05:19,847 [steps/nnet3/chain/train.py:509 - train - INFO ] Iter: 947/1265    Epoch: 2.49/4.0 (62.3% complete)    lr: 0.000477    \n"
     ]
    }
   ],
   "source": [
    "!export CUDA_VISIBLE_DEVICES=0,1,2 && cd chain && make chain |  tee chain.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./steps/nnet3/align.sh --nj 3 --use_gpu true ../train_fbank ../gmm/data/lang exp/chain/tdnn_attend align\n",
      "./steps/nnet3/align.sh: feature type is raw\n",
      "./steps/nnet3/align.sh: aligning data in ../train_fbank using model from exp/chain/tdnn_attend, putting alignments in align\n",
      "./steps/nnet3/align.sh: frame-subsampling-factor is not 1 (so likely a chain system),\n",
      "...  but the scale opts are the defaults.  You probably want\n",
      "--scale-opts '--transition-scale=1.0 --acoustic-scale=1.0 --self-loop-scale=1.0'\n",
      "steps/diagnostic/analyze_alignments.sh --cmd run.pl ../gmm/data/lang align\n",
      "steps/diagnostic/analyze_alignments.sh: see stats in align/log/analyze_alignments.log\n",
      "./steps/nnet3/align.sh: done aligning data.\n"
     ]
    }
   ],
   "source": [
    "!export CUDA_VISIBLE_DEVICES=0,1,2 && cd chain &&./steps/nnet3/align.sh --nj 3 --use_gpu true ../train_fbank ../gmm/data/lang exp/chain/tdnn_attend align"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./steps/get_train_ctm.sh data/train_fbank data/lang align\r\n"
     ]
    }
   ],
   "source": [
    "!cd chain && ./steps/get_train_ctm.sh  data/train_fbank data/lang align"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "warning: discount coeff 1 is out of range: 0\r\n"
     ]
    }
   ],
   "source": [
    "#!sed 's/[^\\s]*\\s//' corpus.seg > corpus_lm.txt\n",
    "!ngram-count -text corpus_lm.txt -order 1 -unk -map-unk \"<UNK>\" -interpolate -lm corpus.lm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "local/make_graph.sh /home1/meichaoyang/Workspace/wake_up/corpus.lm `pwd`/data/lang exp/chain/tdnn_attend `pwd`/data/dict\r\n",
      "gzip: /home1/meichaoyang/Workspace/wake_up/corpus.lm: No such file or directory\r\n",
      "Makefile:18: recipe for target 'make_graph' failed\r\n",
      "make: *** [make_graph] Error 1\r\n"
     ]
    }
   ],
   "source": [
    "!cd chain/ && make make_graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "du: cannot access 'chain/exp': No such file or directory\r\n"
     ]
    }
   ],
   "source": [
    "!du -sh chain/exp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
